Skip to main content

docsrs_cli/
docs_rs.rs

1//! docs.rs operations: readme, get-item, search-in-crate.
2//!
3//! # Extraction posture (Rules Rust — web scraping / data extraction)
4//!
5//! - **Transport** is HTTP GET ([`HttpClient`]); this module only parses bytes.
6//! - **Structure** uses `scraper` (html5ever) + CSS selectors — never regex for
7//!   field extraction from HTML.
8//! - **Sanitization** may use process-static regex only to strip `on*` handlers
9//!   and `javascript:` URLs after DOM script/style removal (XSS hygiene, not
10//!   content discovery).
11//! - **Markdown** conversion via `htmd` on the sanitized fragment.
12//! - **Provenance:** every success payload includes `source_url` (final URL).
13//! - **Encoding:** UTF-8 only after BOM strip ([`crate::http::decode_utf8`]);
14//!   docs.rs / doc.rust-lang.org ship UTF-8. `encoding_rs` multi-charset is OOS.
15//! - **Not a crawl:** no link frontier, no sitemap, no RSS/Atom, no robots meta
16//!   index policy (agent one-shot read, not dataset indexing).
17//!
18//! # Workload classification (Rules Rust — parallelism)
19//!
20//! - **I/O stage:** HTTPS GET via [`HttpClient`] on multi-thread Tokio workers.
21//! - **CPU stage:** HTML scrape + markdown conversion via
22//!   [`ConcurrencyBudget::run_cpu_bound`] (`spawn_blocking` + Semaphore).
23//! - **Hit scan:** large `all.html` candidate lists use `rayon` after a size
24//!   threshold; small lists stay sequential to avoid pool overhead.
25
26use std::collections::HashSet;
27use std::sync::LazyLock;
28
29use rayon::prelude::*;
30use regex::Regex;
31use scraper::{Html, Selector};
32use serde::{Deserialize, Serialize};
33use url::Url;
34
35use crate::config::{
36    HOST_DOC_RUST_LANG_ORG, HOST_DOCS_RS, MAX_SEARCH_IN_CRATE_LIMIT, is_stdlib_crate,
37    stdlib_channel,
38};
39use crate::domain::MatchMode;
40use crate::error::{AppError, AppResult, ErrorKind};
41use crate::http::{HttpClient, content_type_looks_html, decode_utf8};
42use crate::item_kind::{ItemKind, rustc_crate_name};
43
44/// Crate overview (rustdoc crate docblock, not git README).
45#[derive(Debug, Clone, Serialize, Deserialize)]
46pub struct ReadmeData {
47    /// Requested crate name.
48    pub crate_name: String,
49    /// Requested version token.
50    pub version: String,
51    /// Version resolved by docs.rs redirects when known.
52    #[serde(skip_serializing_if = "Option::is_none")]
53    pub resolved_version: Option<String>,
54    /// Markdown extracted from the crate overview.
55    pub markdown: String,
56    /// True when the overview body is empty.
57    pub empty: bool,
58    /// True when output was truncated by `max_output_bytes`.
59    pub truncated: bool,
60    /// Final source URL of the HTML page.
61    pub source_url: String,
62    /// True when the HTTP body was served from the local disk cache.
63    pub cache_hit: bool,
64}
65
66/// Typed item documentation payload.
67#[derive(Debug, Clone, Serialize, Deserialize)]
68pub struct GetItemData {
69    /// Requested crate name.
70    pub crate_name: String,
71    /// Normalized item kind string.
72    pub item_type: String,
73    /// Requested item path.
74    pub item_path: String,
75    /// Last path segment (leaf name) for agent convenience.
76    pub item_name: String,
77    /// Requested version token.
78    pub version: String,
79    /// Version resolved by docs.rs redirects when known.
80    #[serde(skip_serializing_if = "Option::is_none")]
81    pub resolved_version: Option<String>,
82    /// Markdown extracted from the item page.
83    pub markdown: String,
84    /// True when the item body is empty.
85    pub empty: bool,
86    /// True when output was truncated by `max_output_bytes`.
87    pub truncated: bool,
88    /// Final source URL of the HTML page.
89    pub source_url: String,
90    /// Page title when available.
91    pub title: String,
92    /// True when the HTTP body was served from the local disk cache.
93    pub cache_hit: bool,
94    /// How markdown was scoped for associated methods: `method` or `item_page`.
95    #[serde(skip_serializing_if = "Option::is_none")]
96    pub extraction: Option<String>,
97}
98
99/// Single all.html hit.
100#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
101pub struct SearchInCrateHit {
102    /// Symbol or path name from all.html.
103    pub name: String,
104    /// Item kind string (`struct`, `fn`, …).
105    pub kind: String,
106    /// Absolute URL of the item documentation page.
107    pub url: String,
108    /// Match quality score (0 = best). Omitted when query is empty.
109    #[serde(skip_serializing_if = "Option::is_none")]
110    pub score: Option<u8>,
111}
112
113/// search-in-crate result set (`truncated` is true when `total > emitted`).
114#[derive(Debug, Clone, Serialize, Deserialize)]
115pub struct SearchInCrateData {
116    /// Requested crate name.
117    pub crate_name: String,
118    /// Text filter applied to hits.
119    pub query: String,
120    /// Requested version token.
121    pub version: String,
122    /// Echo of the applied item-type filter when set.
123    #[serde(skip_serializing_if = "Option::is_none")]
124    pub item_type: Option<String>,
125    /// Match mode applied (`exact` | `prefix` | `substring`).
126    pub match_mode: String,
127    /// Total classified hits before the limit.
128    pub total: usize,
129    /// Number of hits actually emitted.
130    pub emitted: usize,
131    /// Emitted hits.
132    pub hits: Vec<SearchInCrateHit>,
133    /// True when the hit list was cut by `--limit` (`total > emitted`).
134    pub truncated: bool,
135    /// Final source URL of the all.html page.
136    pub source_url: String,
137    /// True when the HTTP body was served from the local disk cache.
138    pub cache_hit: bool,
139}
140
141fn default_docs_origin(crate_name: &str) -> String {
142    if is_stdlib_crate(crate_name) {
143        format!("https://{HOST_DOC_RUST_LANG_ORG}")
144    } else {
145        format!("https://{HOST_DOCS_RS}")
146    }
147}
148
149/// Build crate index URL with rustc hyphen→underscore segment.
150///
151/// # Errors
152///
153/// Propagates [`ErrorKind::Internal`] from [`readme_url_on_origin`] when the URL is invalid.
154pub fn readme_url(crate_name: &str, version: &str) -> AppResult<Url> {
155    readme_url_on_origin(&default_docs_origin(crate_name), crate_name, version)
156}
157
158/// Build readme URL against a custom origin (wiremock tests).
159///
160/// # Errors
161///
162/// Returns [`ErrorKind::Internal`] when `origin` / path segments do not form a valid URL.
163pub fn readme_url_on_origin(origin: &str, crate_name: &str, version: &str) -> AppResult<Url> {
164    let origin = origin.trim_end_matches('/');
165    let s = if is_stdlib_crate(crate_name) {
166        // doc.rust-lang.org/{channel}/{crate}/index.html
167        let channel = stdlib_channel(version);
168        format!("{origin}/{channel}/{crate_name}/index.html")
169    } else {
170        let rustc = rustc_crate_name(crate_name);
171        format!("{origin}/{crate_name}/{version}/{rustc}/index.html")
172    };
173    Url::parse(&s).map_err(|e| AppError::with_source(ErrorKind::Internal, "invalid readme URL", e))
174}
175
176/// Build rustdoc item or module URL.
177///
178/// # Errors
179///
180/// Propagates [`ErrorKind::InvalidInput`] or [`ErrorKind::Internal`] from
181/// [`get_item_url_on_origin`].
182pub fn get_item_url(
183    crate_name: &str,
184    version: &str,
185    kind: ItemKind,
186    segments: &[String],
187) -> AppResult<Url> {
188    get_item_url_on_origin(
189        &default_docs_origin(crate_name),
190        crate_name,
191        version,
192        kind,
193        segments,
194    )
195}
196
197/// Detect associated method / inherent method path: `Type::method` where Type is
198/// UpperCamel and method starts lowercase (or kind forced as method via parse alias).
199fn is_method_path(kind: ItemKind, segs: &[String]) -> bool {
200    if segs.len() < 2 {
201        return false;
202    }
203    if kind != ItemKind::Fn {
204        return false;
205    }
206    let parent = segs[segs.len() - 2].as_str();
207    let method = segs[segs.len() - 1].as_str();
208    let parent_type = parent
209        .chars()
210        .next()
211        .is_some_and(|c| c.is_ascii_uppercase());
212    let method_fn = method
213        .chars()
214        .next()
215        .is_some_and(|c| c.is_ascii_lowercase() || c == '_');
216    parent_type && method_fn
217}
218
219/// Parent type kinds tried when resolving a method URL (struct first — common case).
220const METHOD_PARENT_KINDS: &[ItemKind] = &[
221    ItemKind::Struct,
222    ItemKind::Enum,
223    ItemKind::Trait,
224    ItemKind::Type,
225    ItemKind::Union,
226];
227
228/// Build get-item URL against a custom origin (wiremock tests).
229///
230/// Associated methods (`Runtime::new`) resolve to the parent type page plus
231/// `#method.{name}` (rustdoc layout). Free functions keep `{kind}.{name}.html`.
232///
233/// # Errors
234///
235/// Returns [`ErrorKind::InvalidInput`] when a non-module path lacks an item name.
236/// Returns [`ErrorKind::Internal`] when the assembled path is not a valid URL.
237pub fn get_item_url_on_origin(
238    origin: &str,
239    crate_name: &str,
240    version: &str,
241    kind: ItemKind,
242    segments: &[String],
243) -> AppResult<Url> {
244    get_item_url_on_origin_with_parent_kind(origin, crate_name, version, kind, segments, None)
245}
246
247/// Like [`get_item_url_on_origin`] but forces the parent type kind for methods.
248pub fn get_item_url_on_origin_with_parent_kind(
249    origin: &str,
250    crate_name: &str,
251    version: &str,
252    kind: ItemKind,
253    segments: &[String],
254    parent_kind_override: Option<ItemKind>,
255) -> AppResult<Url> {
256    let origin = origin.trim_end_matches('/');
257    let rustc_root = rustc_crate_name(crate_name);
258    let segs: Vec<String> = {
259        let mut s = segments.to_vec();
260        if let Some(first) = s.first() {
261            let f = first.as_str();
262            let is_crate_prefix = f == crate_name || f == rustc_root.as_str();
263            if is_crate_prefix && (s.len() >= 2 || kind == ItemKind::Module) {
264                s.remove(0);
265            }
266        }
267        s
268    };
269
270    // Associated method: Type::method → parent page + #method.name
271    if is_method_path(kind, &segs) {
272        let method_name = segs
273            .last()
274            .expect("is_method_path requires >=2 segs")
275            .clone();
276        let parent_name = segs[segs.len() - 2].clone();
277        let parent_kind = parent_kind_override.unwrap_or(ItemKind::Struct);
278        let mod_parts: Vec<String> = if segs.len() == 2 {
279            if is_stdlib_crate(crate_name) {
280                Vec::new()
281            } else {
282                vec![rustc_root.clone()]
283            }
284        } else {
285            let mut m = if is_stdlib_crate(crate_name) {
286                Vec::new()
287            } else {
288                vec![rustc_root.clone()]
289            };
290            for p in &segs[..segs.len() - 2] {
291                m.push(rustc_crate_name(p));
292            }
293            m
294        };
295        let url_str = if is_stdlib_crate(crate_name) {
296            let channel = stdlib_channel(version);
297            if mod_parts.is_empty() {
298                format!(
299                    "{origin}/{channel}/{crate_name}/{}.{}.html#method.{method_name}",
300                    parent_kind.file_prefix(),
301                    parent_name
302                )
303            } else {
304                format!(
305                    "{origin}/{channel}/{crate_name}/{}/{}.{}.html#method.{method_name}",
306                    mod_parts.join("/"),
307                    parent_kind.file_prefix(),
308                    parent_name
309                )
310            }
311        } else if mod_parts.is_empty() {
312            format!(
313                "{origin}/{crate_name}/{version}/{}/{}.{}.html#method.{method_name}",
314                rustc_root,
315                parent_kind.file_prefix(),
316                parent_name
317            )
318        } else {
319            format!(
320                "{origin}/{crate_name}/{version}/{}/{}.{}.html#method.{method_name}",
321                mod_parts.join("/"),
322                parent_kind.file_prefix(),
323                parent_name
324            )
325        };
326        return Url::parse(&url_str)
327            .map_err(|e| AppError::with_source(ErrorKind::Internal, "invalid get-item URL", e));
328    }
329
330    let url_str = if is_stdlib_crate(crate_name) {
331        let channel = stdlib_channel(version);
332        if kind == ItemKind::Module {
333            let mut parts: Vec<String> = Vec::new();
334            for p in &segs {
335                parts.push(rustc_crate_name(p));
336            }
337            if parts.is_empty() {
338                format!("{origin}/{channel}/{crate_name}/index.html")
339            } else {
340                format!(
341                    "{origin}/{channel}/{crate_name}/{}/index.html",
342                    parts.join("/")
343                )
344            }
345        } else {
346            if segs.is_empty() {
347                return Err(AppError::new(
348                    ErrorKind::InvalidInput,
349                    "item path missing item name",
350                ));
351            }
352            let item_name = segs.last().ok_or_else(|| {
353                AppError::new(ErrorKind::InvalidInput, "item path missing item name")
354            })?;
355            let mod_parts: Vec<String> = segs[..segs.len().saturating_sub(1)]
356                .iter()
357                .map(|p| rustc_crate_name(p))
358                .collect();
359            if mod_parts.is_empty() {
360                format!(
361                    "{origin}/{channel}/{crate_name}/{}.{}.html",
362                    kind.file_prefix(),
363                    item_name
364                )
365            } else {
366                format!(
367                    "{origin}/{channel}/{crate_name}/{}/{}.{}.html",
368                    mod_parts.join("/"),
369                    kind.file_prefix(),
370                    item_name
371                )
372            }
373        }
374    } else if kind == ItemKind::Module {
375        let mut parts: Vec<String> = vec![rustc_root];
376        for p in &segs {
377            parts.push(rustc_crate_name(p));
378        }
379        format!(
380            "{origin}/{crate_name}/{version}/{}/index.html",
381            parts.join("/")
382        )
383    } else {
384        if segs.is_empty() {
385            return Err(AppError::new(
386                ErrorKind::InvalidInput,
387                "item path missing item name",
388            ));
389        }
390        let item_name = segs
391            .last()
392            .ok_or_else(|| AppError::new(ErrorKind::InvalidInput, "item path missing item name"))?;
393        let mod_parts: Vec<String> = if segs.len() == 1 {
394            vec![rustc_root]
395        } else {
396            let mut m = vec![rustc_root];
397            for p in &segs[..segs.len() - 1] {
398                m.push(rustc_crate_name(p));
399            }
400            m
401        };
402        format!(
403            "{origin}/{crate_name}/{version}/{}/{}.{}.html",
404            mod_parts.join("/"),
405            kind.file_prefix(),
406            item_name
407        )
408    };
409
410    Url::parse(&url_str)
411        .map_err(|e| AppError::with_source(ErrorKind::Internal, "invalid get-item URL", e))
412}
413
414/// Build all.html index URL.
415///
416/// # Errors
417///
418/// Propagates [`ErrorKind::Internal`] from [`all_html_url_on_origin`].
419pub fn all_html_url(crate_name: &str, version: &str) -> AppResult<Url> {
420    all_html_url_on_origin(&default_docs_origin(crate_name), crate_name, version)
421}
422
423/// Build all.html URL against a custom origin (wiremock tests).
424///
425/// # Errors
426///
427/// Returns [`ErrorKind::Internal`] when `origin` / path segments do not form a valid URL.
428pub fn all_html_url_on_origin(origin: &str, crate_name: &str, version: &str) -> AppResult<Url> {
429    let origin = origin.trim_end_matches('/');
430    let s = if is_stdlib_crate(crate_name) {
431        let channel = stdlib_channel(version);
432        format!("{origin}/{channel}/{crate_name}/all.html")
433    } else {
434        let rustc = rustc_crate_name(crate_name);
435        format!("{origin}/{crate_name}/{version}/{rustc}/all.html")
436    };
437    Url::parse(&s)
438        .map_err(|e| AppError::with_source(ErrorKind::Internal, "invalid all.html URL", e))
439}
440
441#[cfg_attr(not(test), allow(dead_code))]
442fn extract_resolved_version(final_url: &Url, requested: &str) -> Option<String> {
443    extract_resolved_version_for_crate(final_url, requested, None)
444}
445
446/// Resolve version/channel from the final URL, aware of docs.rs vs stdlib layout.
447///
448/// - docs.rs: `/{pkg}/{version}/…` — second path segment is the version
449/// - doc.rust-lang.org: `/{channel}/{crate}/…` — first path segment is channel/version
450fn extract_resolved_version_for_crate(
451    final_url: &Url,
452    requested: &str,
453    crate_name: Option<&str>,
454) -> Option<String> {
455    let host = final_url.host_str().unwrap_or("");
456    let segs: Vec<&str> = final_url
457        .path_segments()?
458        .filter(|s| !s.is_empty())
459        .collect();
460    if segs.is_empty() {
461        return None;
462    }
463
464    let is_stdlib_host = host.eq_ignore_ascii_case(HOST_DOC_RUST_LANG_ORG)
465        || crate_name.is_some_and(is_stdlib_crate)
466        || segs
467            .get(1)
468            .is_some_and(|s| matches!(*s, "std" | "core" | "alloc"));
469
470    if is_stdlib_host {
471        // /{channel}/{crate}/… — never emit crate name as version
472        let channel = segs.first()?;
473        if matches!(*channel, "std" | "core" | "alloc") {
474            return None;
475        }
476        return Some((*channel).to_string());
477    }
478
479    // docs.rs: /{pkg}/{version}/…
480    let ver = segs.get(1)?;
481    if *ver == "latest" {
482        return None;
483    }
484    if requested == "latest" || *ver != requested {
485        return Some((*ver).to_string());
486    }
487    Some((*ver).to_string())
488}
489
490/// Try to scrape a concrete SemVer from docs.rs HTML when the URL still says `/latest/`.
491///
492/// Only accepts versions that belong to `crate_name` (never a dependency hit).
493fn scrape_docs_rs_version_from_html(html: &str, crate_name: &str) -> Option<String> {
494    let crate_name = crate_name.trim();
495    if crate_name.is_empty() || is_stdlib_crate(crate_name) {
496        return None;
497    }
498    // Crate names are already validated (`[A-Za-z][A-Za-z0-9_-]*`); escape for safety.
499    let escaped = regex::escape(crate_name);
500    // Prefer paths for THIS crate: `/tokio/1.53.0/` or `docs.rs/tokio/1.53.0`
501    let re_path = Regex::new(&format!(
502        r"(?i)(?:https?://docs\.rs/|/){escaped}/([0-9]+\.[0-9]+\.[0-9]+(?:-[0-9A-Za-z.-]+)?)\b"
503    ))
504    .ok()?;
505    if let Some(c) = re_path.captures(html) {
506        return c.get(1).map(|m| m.as_str().to_string());
507    }
508    // Title may include "crate version" on some rustdoc skins
509    let re_title = Regex::new(&format!(
510        r"(?i)<title>[^<]*?\b{escaped}\b[^<]*?\b([0-9]+\.[0-9]+\.[0-9]+(?:-[0-9A-Za-z.-]+)?)\b"
511    ))
512    .ok()?;
513    re_title
514        .captures(html)
515        .and_then(|c| c.get(1).map(|m| m.as_str().to_string()))
516}
517
518/// Compiled once: strip `on*` event handlers from HTML fragments (hot path for every HTML→MD).
519static RE_ON_HANDLERS: LazyLock<Regex> = LazyLock::new(|| {
520    Regex::new(r#"(?i)\s+on[a-z]+\s*=\s*("[^"]*"|'[^']*'|[^\s>]+)"#)
521        .expect("hardcoded on* handler strip regex is valid by construction")
522});
523
524/// Compiled once: strip `javascript:` URLs from href/src attributes.
525static RE_JS_URLS: LazyLock<Regex> = LazyLock::new(|| {
526    Regex::new(r#"(?i)(href|src)\s*=\s*["']\s*javascript:[^"']*["']"#)
527        .expect("hardcoded javascript: URL strip regex is valid by construction")
528});
529
530/// Remove script/style nodes and on* handlers before Markdown conversion.
531///
532/// Extraction of doc content uses CSS selectors elsewhere in this module.
533/// Regexes here are **security scrub only** (`on*` / `javascript:`), compiled
534/// once via process-static [`LazyLock`] — not used to scrape fields from HTML.
535pub fn sanitize_html_fragment(html: &str) -> String {
536    let fragment = Html::parse_fragment(html);
537    let drop_sel = Selector::parse("script, style, noscript")
538        .expect("hardcoded scraper selector 'script, style, noscript' is valid by construction");
539    // Prefer reconstructing from root children excluding dropped tags.
540    if let Some(root) = fragment.tree.root().children().next() {
541        let _ = root;
542    }
543    // Collect HTML of non-script/style nodes by stripping matches from a working copy.
544    // Capacity ≈ input length: script/style removals only shrink the buffer.
545    // Precondition: `html` is already body-capped (HARD_MAX_BODY_BYTES) and resident in RAM.
546    let mut out = String::with_capacity(html.len());
547    out.push_str(html);
548    for el in fragment.select(&drop_sel) {
549        let tag_html = el.html();
550        out = out.replace(&tag_html, "");
551    }
552    // Strip event handlers: onload=, onclick=, etc.
553    out = RE_ON_HANDLERS.replace_all(&out, "").into_owned();
554    // Drop javascript: URLs in href/src
555    out = RE_JS_URLS.replace_all(&out, "").into_owned();
556    out
557}
558
559fn html_to_markdown(html: &str) -> AppResult<String> {
560    let cleaned = sanitize_html_fragment(html);
561    let md = htmd::convert(&cleaned).map_err(|e| {
562        AppError::new(
563            ErrorKind::Parse,
564            format!("HTML to Markdown conversion failed: {e}"),
565        )
566    })?;
567    Ok(scrub_rustdoc_chrome(&md))
568}
569
570/// Strip rustdoc UI chrome that pollutes LLM context (`§`, "Copy item path", …).
571pub fn scrub_rustdoc_chrome(md: &str) -> String {
572    let mut out = md.to_string();
573    // Common rustdoc heading anchor glyph left by htmd.
574    out = out.replace('§', "");
575    // "Copy item path" UI control sometimes survives as text.
576    out = out.replace("Copy item path", "");
577    // Collapse accidental double spaces left after removals (not full reflow).
578    while out.contains("  ") {
579        out = out.replace("  ", " ");
580    }
581    out
582}
583
584fn first_inner_html(document: &Html, selectors: &[&str]) -> Option<String> {
585    for sel in selectors {
586        if let Ok(s) = Selector::parse(sel)
587            && let Some(n) = document.select(&s).next()
588        {
589            let inner = n.inner_html();
590            if !inner.trim().is_empty() {
591                return Some(inner);
592            }
593        }
594    }
595    None
596}
597
598/// Fetch and convert crate overview docblock (production docs.rs).
599///
600/// # Errors
601///
602/// Propagates URL, HTTP, decode, and HTML conversion failures from
603/// [`fetch_readme_on_origin`].
604pub async fn fetch_readme(
605    http: &HttpClient,
606    crate_name: &str,
607    version: &str,
608) -> AppResult<ReadmeData> {
609    fetch_readme_on_origin(
610        http,
611        &format!("https://{HOST_DOCS_RS}"),
612        crate_name,
613        version,
614    )
615    .await
616}
617
618/// Fetch readme against a configurable origin (offline mocks).
619///
620/// # Errors
621///
622/// Returns [`ErrorKind::Internal`] when the origin URL is invalid.
623/// Propagates fetch failures from [`fetch_readme_at`].
624pub async fn fetch_readme_on_origin(
625    http: &HttpClient,
626    origin: &str,
627    crate_name: &str,
628    version: &str,
629) -> AppResult<ReadmeData> {
630    let url = readme_url_on_origin(origin, crate_name, version)?;
631    fetch_readme_at(http, crate_name, version, &url).await
632}
633
634/// Fetch readme from a prebuilt URL (wiremock).
635///
636/// # Errors
637///
638/// Propagates HTTP errors from [`HttpClient::get_html`].
639/// Maps non-success statuses via [`AppError::from_http_status`].
640/// Returns [`ErrorKind::Parse`] on non-UTF-8 bodies or HTML→Markdown failure.
641pub async fn fetch_readme_at(
642    http: &HttpClient,
643    crate_name: &str,
644    version: &str,
645    url: &Url,
646) -> AppResult<ReadmeData> {
647    let resp = http.get_html(url).await?;
648    if resp.status.as_u16() == 404 {
649        return Err(AppError::from_http_status(
650            404,
651            &format!("crate or version not found: {crate_name}@{version}"),
652        ));
653    }
654    if !resp.status.is_success() {
655        return Err(AppError::from_http_status(
656            resp.status.as_u16(),
657            "docs.rs readme",
658        ));
659    }
660    if resp.content_type.is_some() && !content_type_looks_html(resp.content_type.as_deref()) {
661        tracing::warn!(content_type = ?resp.content_type, "unexpected Content-Type from docs.rs");
662    }
663
664    let body = resp.body.clone();
665    let (markdown, empty) = http
666        .budget()
667        .run_cpu_bound(move || {
668            let text = decode_utf8(&body)?;
669            extract_readme_markdown_from_html(&text)
670        })
671        .await?;
672    let mut resolved =
673        extract_resolved_version_for_crate(&resp.final_url, version, Some(crate_name));
674    if resolved.is_none() {
675        // Body still available only as markdown path; re-decode for scrape when latest.
676        if let Ok(text) = decode_utf8(&resp.body) {
677            resolved = scrape_docs_rs_version_from_html(&text, crate_name);
678        }
679    }
680    Ok(ReadmeData {
681        crate_name: crate_name.to_string(),
682        version: version.to_string(),
683        resolved_version: resolved,
684        markdown,
685        empty,
686        truncated: false,
687        source_url: resp.final_url.to_string(),
688        cache_hit: resp.cache_hit,
689    })
690}
691
692/// Fetch and convert a typed rustdoc item page (production docs.rs).
693///
694/// # Errors
695///
696/// Propagates URL, HTTP, decode, and HTML conversion failures from
697/// [`fetch_item_on_origin`].
698pub async fn fetch_item(
699    http: &HttpClient,
700    crate_name: &str,
701    version: &str,
702    kind: ItemKind,
703    segments: &[String],
704) -> AppResult<GetItemData> {
705    fetch_item_on_origin(
706        http,
707        &format!("https://{HOST_DOCS_RS}"),
708        crate_name,
709        version,
710        kind,
711        segments,
712    )
713    .await
714}
715/// Fetch item against a configurable origin (offline mocks).
716///
717/// For associated methods, tries parent type pages (struct → enum → trait → …)
718/// until one returns HTTP 200.
719///
720/// # Errors
721///
722/// Propagates URL build errors from [`get_item_url_on_origin`] and fetch errors
723/// from [`fetch_item_at`].
724pub async fn fetch_item_on_origin(
725    http: &HttpClient,
726    origin: &str,
727    crate_name: &str,
728    version: &str,
729    kind: ItemKind,
730    segments: &[String],
731) -> AppResult<GetItemData> {
732    // Strip crate prefix the same way URL builder does for method detection.
733    let rustc_root = rustc_crate_name(crate_name);
734    let segs_for_detect: Vec<String> = {
735        let mut s = segments.to_vec();
736        if let Some(first) = s.first() {
737            let f = first.as_str();
738            let is_crate_prefix = f == crate_name || f == rustc_root.as_str();
739            if is_crate_prefix && (s.len() >= 2 || kind == ItemKind::Module) {
740                s.remove(0);
741            }
742        }
743        s
744    };
745
746    if is_method_path(kind, &segs_for_detect) {
747        let mut last_err: Option<AppError> = None;
748        for parent_kind in METHOD_PARENT_KINDS {
749            let url = get_item_url_on_origin_with_parent_kind(
750                origin,
751                crate_name,
752                version,
753                kind,
754                segments,
755                Some(*parent_kind),
756            )?;
757            match fetch_item_at(http, crate_name, version, kind, segments, &url).await {
758                Ok(data) => return Ok(data),
759                Err(e) if e.kind() == ErrorKind::NotFound => {
760                    last_err = Some(e);
761                    continue;
762                }
763                Err(e) => return Err(e),
764            }
765        }
766        return Err(last_err.unwrap_or_else(|| {
767            AppError::new(ErrorKind::NotFound, "method parent type page not found")
768        }));
769    }
770
771    let url = get_item_url_on_origin(origin, crate_name, version, kind, segments)?;
772    fetch_item_at(http, crate_name, version, kind, segments, &url).await
773}
774
775/// Fetch item from a prebuilt URL (wiremock).
776///
777/// # Errors
778///
779/// Propagates HTTP errors from [`HttpClient::get_html`].
780/// Maps non-success statuses via [`AppError::from_http_status`].
781/// Returns [`ErrorKind::Parse`] on non-UTF-8 bodies or HTML→Markdown failure.
782pub async fn fetch_item_at(
783    http: &HttpClient,
784    crate_name: &str,
785    version: &str,
786    kind: ItemKind,
787    segments: &[String],
788    url: &Url,
789) -> AppResult<GetItemData> {
790    // HTTP does not send fragments; strip for request, keep for source_url.
791    let mut fetch_url = url.clone();
792    let fragment = fetch_url.fragment().map(str::to_string);
793    fetch_url.set_fragment(None);
794
795    let resp = http.get_html(&fetch_url).await?;
796    if resp.status.as_u16() == 404 {
797        return Err(AppError::from_http_status(
798            404,
799            &format!("item not found at {url}"),
800        ));
801    }
802    if !resp.status.is_success() {
803        return Err(AppError::from_http_status(
804            resp.status.as_u16(),
805            "docs.rs get-item",
806        ));
807    }
808
809    let method_anchor = fragment
810        .as_deref()
811        .and_then(|f| f.strip_prefix("method."))
812        .map(str::to_string);
813    let body = resp.body.clone();
814    let method_anchor_cpu = method_anchor.clone();
815    let ((markdown, empty), extraction) = http
816        .budget()
817        .run_cpu_bound(move || {
818            let text = decode_utf8(&body)?;
819            if let Some(ref m) = method_anchor_cpu {
820                let (md, empty, scope) = extract_method_markdown_scoped(&text, m)?;
821                Ok(((md, empty), Some(scope.to_string())))
822            } else {
823                let (md, empty) = extract_item_markdown_from_html(&text)?;
824                Ok(((md, empty), None))
825            }
826        })
827        .await?;
828    let item_path = segments.join("::");
829    let item_name = segments
830        .last()
831        .cloned()
832        .unwrap_or_else(|| item_path.clone());
833    let title = format!("{item_path} ({})", kind.as_str());
834    let mut resolved =
835        extract_resolved_version_for_crate(&resp.final_url, version, Some(crate_name));
836    if resolved.is_none() {
837        if let Ok(text) = decode_utf8(&resp.body) {
838            resolved = scrape_docs_rs_version_from_html(&text, crate_name);
839        }
840    }
841
842    let mut source_url = resp.final_url.clone();
843    if let Some(f) = fragment {
844        source_url.set_fragment(Some(&f));
845    }
846
847    Ok(GetItemData {
848        crate_name: crate_name.to_string(),
849        item_type: kind.as_str().to_string(),
850        item_path,
851        item_name,
852        version: version.to_string(),
853        resolved_version: resolved,
854        markdown,
855        empty,
856        truncated: false,
857        source_url: source_url.to_string(),
858        title,
859        cache_hit: resp.cache_hit,
860        extraction,
861    })
862}
863/// Parse all.html and filter symbols (production docs.rs).
864///
865/// # Errors
866///
867/// Propagates URL, HTTP, decode, and parse failures from
868/// [`search_in_crate_on_origin`].
869pub async fn search_in_crate(
870    http: &HttpClient,
871    crate_name: &str,
872    version: &str,
873    query: &str,
874    item_type: Option<ItemKind>,
875    limit: u32,
876    match_mode: MatchMode,
877) -> AppResult<SearchInCrateData> {
878    search_in_crate_on_origin(
879        http,
880        &format!("https://{HOST_DOCS_RS}"),
881        crate_name,
882        version,
883        query,
884        item_type,
885        limit,
886        match_mode,
887    )
888    .await
889}
890
891/// search-in-crate against a configurable origin (offline mocks).
892///
893/// # Errors
894///
895/// Propagates URL build errors from [`all_html_url_on_origin`] and fetch errors
896/// from [`search_in_crate_at`].
897#[allow(clippy::too_many_arguments)]
898pub async fn search_in_crate_on_origin(
899    http: &HttpClient,
900    origin: &str,
901    crate_name: &str,
902    version: &str,
903    query: &str,
904    item_type: Option<ItemKind>,
905    limit: u32,
906    match_mode: MatchMode,
907) -> AppResult<SearchInCrateData> {
908    let url = all_html_url_on_origin(origin, crate_name, version)?;
909    search_in_crate_at(
910        http, crate_name, version, query, item_type, limit, match_mode, &url,
911    )
912    .await
913}
914
915/// search-in-crate against a prebuilt all.html URL (wiremock).
916///
917/// # Errors
918///
919/// Propagates HTTP errors from [`HttpClient::get_html`].
920/// Maps non-success statuses via [`AppError::from_http_status`].
921/// Returns [`ErrorKind::Parse`] on non-UTF-8 bodies or href join failures.
922/// Propagates parse errors from [`search_in_crate_from_html`].
923#[allow(clippy::too_many_arguments)]
924pub async fn search_in_crate_at(
925    http: &HttpClient,
926    crate_name: &str,
927    version: &str,
928    query: &str,
929    item_type: Option<ItemKind>,
930    limit: u32,
931    match_mode: MatchMode,
932    url: &Url,
933) -> AppResult<SearchInCrateData> {
934    let resp = http.get_html(url).await?;
935    if resp.status.as_u16() == 404 {
936        return Err(AppError::from_http_status(
937            404,
938            &format!("all.html not found for {crate_name}@{version}"),
939        ));
940    }
941    if !resp.status.is_success() {
942        return Err(AppError::from_http_status(
943            resp.status.as_u16(),
944            "docs.rs all.html",
945        ));
946    }
947
948    let body = resp.body.clone();
949    let source_url = resp.final_url.to_string();
950    let cache_hit = resp.cache_hit;
951    let crate_name = crate_name.to_string();
952    let version = version.to_string();
953    let query = query.to_string();
954    http.budget()
955        .run_cpu_bound(move || {
956            let text = decode_utf8(&body)?;
957            search_in_crate_from_html(
958                &text,
959                &crate_name,
960                &version,
961                &query,
962                item_type,
963                limit,
964                match_mode,
965                &source_url,
966                cache_hit,
967            )
968        })
969        .await
970}
971
972/// Join relative/absolute/full href against crate rustdoc base.
973///
974/// # Errors
975///
976/// Returns [`ErrorKind::Parse`] when a relative or absolute-path href cannot be joined.
977pub fn join_href(base: &Url, href: &str) -> AppResult<String> {
978    if href.starts_with("http://") || href.starts_with("https://") {
979        return Ok(href.to_string());
980    }
981    if href.starts_with('/') {
982        let joined = Url::parse(&format!("https://docs.rs{href}")).map_err(|e| {
983            AppError::with_source(ErrorKind::Parse, "failed to join absolute path href", e)
984        })?;
985        return Ok(joined.to_string());
986    }
987    base.join(href)
988        .map(|u| u.to_string())
989        .map_err(|e| AppError::with_source(ErrorKind::Parse, "failed to join relative href", e))
990}
991
992/// Minimum candidate count before `rayon` fan-out (overhead below this is pure loss).
993const RAYON_HIT_THRESHOLD: usize = 64;
994
995/// One classified hit candidate before dedup/limit (index preserves document order).
996/// One classified hit candidate before dedup/limit: (index, name, kind, url, score).
997type ClassifiedHit = (usize, String, ItemKind, String, u8);
998
999/// Pure parse of all.html body for offline tests and CPU workers.
1000///
1001/// Large candidate lists use `rayon`; small lists stay sequential.
1002/// Results are scored and sorted (exact leaf first) before applying `limit`.
1003///
1004/// # Errors
1005///
1006/// Returns [`ErrorKind::Internal`] when the synthetic base URL is invalid.
1007/// Returns [`ErrorKind::Parse`] when an href cannot be joined via [`join_href`].
1008pub fn parse_all_html_hits(
1009    html: &str,
1010    crate_name: &str,
1011    version: &str,
1012    query: &str,
1013    item_type: Option<ItemKind>,
1014    limit: usize,
1015    match_mode: MatchMode,
1016) -> AppResult<Vec<SearchInCrateHit>> {
1017    if limit == 0 {
1018        return Ok(Vec::new());
1019    }
1020    let document = Html::parse_document(html);
1021    let a_sel = Selector::parse("#main-content a")
1022        .expect("hardcoded scraper selector '#main-content a' is valid by construction");
1023    let rustc = rustc_crate_name(crate_name);
1024    let base = Url::parse(&format!(
1025        "https://{HOST_DOCS_RS}/{crate_name}/{version}/{rustc}/"
1026    ))
1027    .map_err(|e| AppError::with_source(ErrorKind::Internal, "invalid all.html base", e))?;
1028    let q = query.trim().to_ascii_lowercase();
1029
1030    let candidates: Vec<(String, String)> = document
1031        .select(&a_sel)
1032        .filter_map(|a| {
1033            let name = a.text().collect::<String>().trim().to_string();
1034            if name.is_empty() {
1035                return None;
1036            }
1037            let href = a.value().attr("href")?.to_string();
1038            if href.is_empty() {
1039                return None;
1040            }
1041            Some((name, href))
1042        })
1043        .collect();
1044
1045    if candidates.len() < RAYON_HIT_THRESHOLD {
1046        return filter_hits_sequential(candidates, &base, &q, item_type, limit, match_mode);
1047    }
1048    filter_hits_parallel(candidates, &base, &q, item_type, limit, match_mode)
1049}
1050
1051fn filter_hits_sequential(
1052    candidates: Vec<(String, String)>,
1053    base: &Url,
1054    q: &str,
1055    item_type: Option<ItemKind>,
1056    limit: usize,
1057    match_mode: MatchMode,
1058) -> AppResult<Vec<SearchInCrateHit>> {
1059    let mut rows: Vec<ClassifiedHit> = Vec::new();
1060    for (idx, (name, href)) in candidates.into_iter().enumerate() {
1061        if let Some(hit) = classify_hit_row(idx, name, &href, base, q, item_type, match_mode)? {
1062            rows.push(hit);
1063        }
1064    }
1065    finalize_hits(rows, limit)
1066}
1067
1068fn filter_hits_parallel(
1069    candidates: Vec<(String, String)>,
1070    base: &Url,
1071    q: &str,
1072    item_type: Option<ItemKind>,
1073    limit: usize,
1074    match_mode: MatchMode,
1075) -> AppResult<Vec<SearchInCrateHit>> {
1076    let mapped: AppResult<Vec<Option<ClassifiedHit>>> = candidates
1077        .into_par_iter()
1078        .enumerate()
1079        .map(|(idx, (name, href))| {
1080            classify_hit_row(idx, name, &href, base, q, item_type, match_mode)
1081        })
1082        .collect();
1083    let rows: Vec<ClassifiedHit> = mapped?.into_iter().flatten().collect();
1084    finalize_hits(rows, limit)
1085}
1086
1087fn finalize_hits(mut rows: Vec<ClassifiedHit>, limit: usize) -> AppResult<Vec<SearchInCrateHit>> {
1088    // Score ascending, then original document order for stability.
1089    rows.sort_by(|a, b| a.4.cmp(&b.4).then_with(|| a.0.cmp(&b.0)));
1090    let mut hits = Vec::with_capacity(limit.min(256));
1091    let mut seen: HashSet<(String, ItemKind)> = HashSet::new();
1092    for (_idx, name, kind, abs, score) in rows {
1093        if !seen.insert((name.clone(), kind)) {
1094            continue;
1095        }
1096        hits.push(SearchInCrateHit {
1097            name,
1098            kind: kind.as_str().to_string(),
1099            url: abs,
1100            score: Some(score),
1101        });
1102        if hits.len() >= limit {
1103            break;
1104        }
1105    }
1106    Ok(hits)
1107}
1108
1109fn classify_hit_row(
1110    idx: usize,
1111    name: String,
1112    href: &str,
1113    base: &Url,
1114    q: &str,
1115    item_type: Option<ItemKind>,
1116    match_mode: MatchMode,
1117) -> AppResult<Option<ClassifiedHit>> {
1118    let Some(kind) = ItemKind::from_href(href) else {
1119        return Ok(None);
1120    };
1121    if let Some(filter) = item_type
1122        && kind != filter
1123    {
1124        return Ok(None);
1125    }
1126    let Some(score) = match_mode.score(&name, q) else {
1127        return Ok(None);
1128    };
1129    let abs = join_href(base, href)?;
1130    Ok(Some((idx, name, kind, abs, score)))
1131}
1132/// Extract readme markdown from raw HTML (offline tests / pure path).
1133///
1134/// # Errors
1135///
1136/// Returns [`ErrorKind::Parse`] when HTML→Markdown conversion fails.
1137pub fn extract_readme_markdown_from_html(html: &str) -> AppResult<(String, bool)> {
1138    let document = Html::parse_document(html);
1139    let content = first_inner_html(
1140        &document,
1141        &[
1142            ".rustdoc .docblock",
1143            "#main-content .docblock",
1144            ".docblock",
1145            ".rustdoc-main .item-decl",
1146            ".item-decl",
1147        ],
1148    );
1149    match content {
1150        Some(h) => Ok((html_to_markdown(&h)?, false)),
1151        None => Ok((String::new(), true)),
1152    }
1153}
1154
1155/// Extract get-item markdown from raw HTML (offline tests / pure path).
1156///
1157/// # Errors
1158///
1159/// Returns [`ErrorKind::Parse`] when HTML→Markdown conversion fails.
1160pub fn extract_item_markdown_from_html(html: &str) -> AppResult<(String, bool)> {
1161    let document = Html::parse_document(html);
1162    let content = first_inner_html(&document, &["#main-content", ".item-decl", ".docblock"])
1163        .unwrap_or_default();
1164    let empty = content.trim().is_empty();
1165    if empty {
1166        Ok((String::new(), true))
1167    } else {
1168        Ok((html_to_markdown(&content)?, false))
1169    }
1170}
1171
1172/// Extract markdown for a single associated method by rustdoc `id="method.X"`.
1173///
1174/// Locates the method anchor and prefers the enclosing `details.method-toggle`
1175/// (signature + docblock). Falls back to the full item page when the anchor is
1176/// missing (`extraction` = `item_page`).
1177///
1178/// # Errors
1179///
1180/// Returns [`ErrorKind::Parse`] when HTML→Markdown conversion fails.
1181pub fn extract_method_markdown_from_html(
1182    html: &str,
1183    method_name: &str,
1184) -> AppResult<(String, bool)> {
1185    let (md, empty, _) = extract_method_markdown_scoped(html, method_name)?;
1186    Ok((md, empty))
1187}
1188
1189/// Like [`extract_method_markdown_from_html`] but reports extraction scope.
1190///
1191/// Returns `(markdown, empty, scope)` where `scope` is `"method"` or `"item_page"`.
1192///
1193/// # Errors
1194///
1195/// Returns [`ErrorKind::Parse`] when HTML→Markdown conversion fails.
1196pub fn extract_method_markdown_scoped(
1197    html: &str,
1198    method_name: &str,
1199) -> AppResult<(String, bool, &'static str)> {
1200    let want_id = format!("method.{method_name}");
1201    let document = Html::parse_document(html);
1202    // Attribute selector avoids CSS id issues with dots in method.X.
1203    let sel = match Selector::parse(&format!(r#"[id="{want_id}"]"#)) {
1204        Ok(s) => s,
1205        Err(_) => {
1206            let (md, empty) = extract_item_markdown_from_html(html)?;
1207            return Ok((md, empty, "item_page"));
1208        }
1209    };
1210    if let Some(anchor) = document.select(&sel).next() {
1211        let frag_html = method_container_html(anchor);
1212        let md = html_to_markdown(&frag_html)?;
1213        let empty = md.trim().is_empty();
1214        if !empty {
1215            return Ok((md, empty, "method"));
1216        }
1217    }
1218    let (md, empty) = extract_item_markdown_from_html(html)?;
1219    Ok((md, empty, "item_page"))
1220}
1221
1222/// Prefer the enclosing rustdoc `details.method-toggle` (signature + docs).
1223fn method_container_html(anchor: scraper::ElementRef<'_>) -> String {
1224    let mut node = anchor.parent();
1225    while let Some(n) = node {
1226        if let Some(el) = scraper::ElementRef::wrap(n) {
1227            if el.value().name() == "details" {
1228                return el.html();
1229            }
1230            node = el.parent();
1231        } else {
1232            break;
1233        }
1234    }
1235    // Fall back to the section/element that carries the method id.
1236    anchor.html()
1237}
1238
1239/// Build SearchInCrateData from HTML body without network (offline tests).
1240///
1241/// `--limit 0` is honoured: `emitted = 0`, `hits = []`, and `truncated` is true
1242/// when the unfiltered total is greater than zero. Values above
1243/// [`MAX_SEARCH_IN_CRATE_LIMIT`] are capped.
1244///
1245/// # Errors
1246///
1247/// Propagates parse failures from [`parse_all_html_hits`].
1248#[allow(clippy::too_many_arguments)]
1249pub fn search_in_crate_from_html(
1250    html: &str,
1251    crate_name: &str,
1252    version: &str,
1253    query: &str,
1254    item_type: Option<ItemKind>,
1255    limit: u32,
1256    match_mode: MatchMode,
1257    source_url: &str,
1258    cache_hit: bool,
1259) -> AppResult<SearchInCrateData> {
1260    let limit = (limit as usize).min(MAX_SEARCH_IN_CRATE_LIMIT as usize);
1261    let all = parse_all_html_hits(
1262        html,
1263        crate_name,
1264        version,
1265        query,
1266        item_type,
1267        usize::MAX,
1268        match_mode,
1269    )?;
1270    let total = all.len();
1271    let hits: Vec<_> = all.into_iter().take(limit).collect();
1272    let emitted = hits.len();
1273    // Empty query: scores are 0 — omit score noise on wire for cleaner lists.
1274    let hits = if query.trim().is_empty() {
1275        hits.into_iter()
1276            .map(|mut h| {
1277                h.score = None;
1278                h
1279            })
1280            .collect()
1281    } else {
1282        hits
1283    };
1284    Ok(SearchInCrateData {
1285        crate_name: crate_name.to_string(),
1286        query: query.to_string(),
1287        version: version.to_string(),
1288        item_type: item_type.map(|k| k.as_str().to_string()),
1289        match_mode: match_mode.as_str().to_string(),
1290        total,
1291        emitted,
1292        hits,
1293        truncated: total > emitted,
1294        source_url: source_url.to_string(),
1295        cache_hit,
1296    })
1297}
1298#[cfg(test)]
1299mod tests {
1300    use super::*;
1301
1302    #[test]
1303    fn readme_url_hyphen() {
1304        let u = readme_url("async-trait", "latest").unwrap();
1305        assert_eq!(
1306            u.as_str(),
1307            "https://docs.rs/async-trait/latest/async_trait/index.html"
1308        );
1309    }
1310
1311    #[test]
1312    fn get_item_nested() {
1313        let segs = vec!["tokio".into(), "runtime".into(), "Runtime".into()];
1314        let u = get_item_url("tokio", "latest", ItemKind::Struct, &segs).unwrap();
1315        assert!(u.as_str().contains("/tokio/runtime/struct.Runtime.html"));
1316    }
1317
1318    #[test]
1319    fn get_item_without_crate_prefix() {
1320        let segs = vec!["Parser".into()];
1321        let u = get_item_url("clap", "latest", ItemKind::Trait, &segs).unwrap();
1322        assert!(
1323            u.as_str().ends_with("/clap/trait.Parser.html")
1324                || u.path().ends_with("/clap/trait.Parser.html")
1325        );
1326    }
1327
1328    #[test]
1329    fn get_item_keeps_single_segment_equal_to_crate_name() {
1330        // Attribute/macro items often share the rustc crate name.
1331        let segs = vec!["async_trait".into()];
1332        let u = get_item_url("async-trait", "latest", ItemKind::Attribute, &segs).unwrap();
1333        assert!(
1334            u.as_str()
1335                .ends_with("/async-trait/latest/async_trait/attr.async_trait.html"),
1336            "url={u}"
1337        );
1338    }
1339
1340    #[test]
1341    fn get_item_module_single_segment_crate_name_is_root() {
1342        let segs = vec!["async_trait".into()];
1343        let u = get_item_url("async-trait", "latest", ItemKind::Module, &segs).unwrap();
1344        assert!(
1345            u.as_str()
1346                .ends_with("/async-trait/latest/async_trait/index.html"),
1347            "url={u}"
1348        );
1349    }
1350
1351    #[test]
1352    fn join_relative() {
1353        let base = Url::parse("https://docs.rs/serde/latest/serde/").unwrap();
1354        let j = join_href(&base, "de/trait.Deserialize.html").unwrap();
1355        assert!(j.contains("/serde/de/trait.Deserialize.html"));
1356    }
1357
1358    #[test]
1359    fn join_absolute_path_and_fragment() {
1360        let base = Url::parse("https://docs.rs/serde/latest/serde/").unwrap();
1361        let j = join_href(&base, "/serde/latest/serde/struct.Error.html").unwrap();
1362        assert!(j.starts_with("https://docs.rs/serde/"));
1363        let f = join_href(&base, "struct.Error.html#method.to_string").unwrap();
1364        assert!(f.contains("#method.to_string"));
1365    }
1366
1367    #[test]
1368    fn sanitize_strips_script() {
1369        let raw = r#"<div>ok<script>alert(1)</script><p onclick="x()">hi</p></div>"#;
1370        let s = sanitize_html_fragment(raw);
1371        assert!(!s.contains("script"));
1372        assert!(!s.contains("onclick"));
1373        assert!(s.contains("hi") || s.contains("ok"));
1374    }
1375
1376    #[test]
1377    fn readme_fallback_item_decl_converts() {
1378        let html = r#"<!DOCTYPE html><html><body class="rustdoc-main">
1379        <div class="item-decl"><pre>pub use foo;</pre></div>
1380        </body></html>"#;
1381        let (md, empty) = extract_readme_markdown_from_html(html).unwrap();
1382        assert!(!empty);
1383        assert!(!md.trim().is_empty());
1384    }
1385
1386    #[test]
1387    fn all_html_order_stable() {
1388        let html = r#"<!DOCTYPE html><html><body>
1389        <div id="main-content">
1390          <a href="struct.A.html">A</a>
1391          <a href="trait.B.html">B</a>
1392          <a href="fn.c.html">c</a>
1393          <a href="union.U.html">U</a>
1394          <a href="attr.x.html">x</a>
1395          <a href="derive.D.html">D</a>
1396        </div></body></html>"#;
1397        let hits = parse_all_html_hits(
1398            html,
1399            "demo",
1400            "1.0.0",
1401            "",
1402            None,
1403            100,
1404            crate::domain::MatchMode::Prefix,
1405        )
1406        .unwrap();
1407        assert_eq!(hits.len(), 6);
1408        assert_eq!(hits[0].name, "A");
1409        assert_eq!(hits[0].kind, "struct");
1410        assert_eq!(hits[3].kind, "union");
1411        assert_eq!(hits[4].kind, "attribute");
1412        assert_eq!(hits[5].kind, "derive");
1413    }
1414
1415    #[test]
1416    fn std_core_alloc_urls() {
1417        let u = readme_url("std", "latest").unwrap();
1418        assert_eq!(
1419            u.as_str(),
1420            "https://doc.rust-lang.org/stable/std/index.html"
1421        );
1422        let u = get_item_url(
1423            "std",
1424            "latest",
1425            ItemKind::Struct,
1426            &["option".into(), "Option".into()],
1427        )
1428        .unwrap();
1429        assert_eq!(
1430            u.as_str(),
1431            "https://doc.rust-lang.org/stable/std/option/struct.Option.html"
1432        );
1433        let a = all_html_url("core", "nightly").unwrap();
1434        assert_eq!(
1435            a.as_str(),
1436            "https://doc.rust-lang.org/nightly/core/all.html"
1437        );
1438    }
1439
1440    #[test]
1441    fn extract_item_from_fixture() {
1442        let html = include_str!("../tests/fixtures/docs_rs/get_item_main.html");
1443        let (md, empty) = extract_item_markdown_from_html(html).unwrap();
1444        assert!(!empty);
1445        assert!(md.contains("Runtime") || md.contains("Tokio") || md.contains("runtime"));
1446    }
1447
1448    #[test]
1449    fn extract_readme_primary_docblock_fixture() {
1450        let html = include_str!("../tests/fixtures/docs_rs/readme_docblock.html");
1451        let (md, empty) = extract_readme_markdown_from_html(html).unwrap();
1452        assert!(!empty);
1453        assert!(!md.to_ascii_lowercase().contains("alert"));
1454    }
1455
1456    #[test]
1457    fn search_in_crate_from_html_limit_and_filter() {
1458        let html = include_str!("../tests/fixtures/docs_rs/all_html_sample.html");
1459        let data = search_in_crate_from_html(
1460            html,
1461            "demo",
1462            "1.0.0",
1463            "",
1464            Some(ItemKind::Struct),
1465            1,
1466            MatchMode::Prefix,
1467            "https://docs.rs/demo/1.0.0/demo/all.html",
1468            false,
1469        )
1470        .unwrap();
1471        assert_eq!(data.total, 1);
1472        assert_eq!(data.emitted, 1);
1473        assert_eq!(data.hits[0].kind, "struct");
1474        assert!(!data.truncated);
1475    }
1476
1477    #[test]
1478    fn search_in_crate_truncated_when_limit_cuts_hits() {
1479        let html = include_str!("../tests/fixtures/docs_rs/all_html_sample.html");
1480        let data = search_in_crate_from_html(
1481            html,
1482            "demo",
1483            "1.0.0",
1484            "",
1485            None,
1486            2,
1487            MatchMode::Prefix,
1488            "https://docs.rs/demo/1.0.0/demo/all.html",
1489            false,
1490        )
1491        .unwrap();
1492        assert!(data.total > 2, "fixture must have more than 2 hits");
1493        assert_eq!(data.emitted, 2);
1494        assert!(data.truncated);
1495        let full = search_in_crate_from_html(
1496            html,
1497            "demo",
1498            "1.0.0",
1499            "",
1500            None,
1501            1000,
1502            MatchMode::Prefix,
1503            "https://docs.rs/demo/1.0.0/demo/all.html",
1504            false,
1505        )
1506        .unwrap();
1507        assert_eq!(full.total, full.emitted);
1508        assert!(!full.truncated);
1509    }
1510
1511    #[test]
1512    fn search_in_crate_limit_zero_emits_nothing() {
1513        let html = include_str!("../tests/fixtures/docs_rs/all_html_sample.html");
1514        let data = search_in_crate_from_html(
1515            html,
1516            "demo",
1517            "1.0.0",
1518            "",
1519            None,
1520            0,
1521            MatchMode::Prefix,
1522            "https://docs.rs/demo/1.0.0/demo/all.html",
1523            false,
1524        )
1525        .unwrap();
1526        assert!(data.total > 0, "fixture must have hits for total");
1527        assert_eq!(data.emitted, 0);
1528        assert!(data.hits.is_empty());
1529        assert!(data.truncated);
1530    }
1531
1532    #[test]
1533    fn serialize_exact_match_beats_deserializer() {
1534        let html = r#"<!DOCTYPE html><html><body>
1535        <div id="main-content">
1536          <a href="trait.Deserialize.html">Deserialize</a>
1537          <a href="trait.Serialize.html">Serialize</a>
1538          <a href="struct.Deserializer.html">Deserializer</a>
1539        </div></body></html>"#;
1540        let hits = parse_all_html_hits(
1541            html,
1542            "serde",
1543            "1.0.0",
1544            "Serialize",
1545            Some(ItemKind::Trait),
1546            10,
1547            MatchMode::Prefix,
1548        )
1549        .unwrap();
1550        assert_eq!(hits[0].name, "Serialize");
1551        assert!(hits.iter().all(|h| h.name == "Serialize"));
1552    }
1553
1554    #[test]
1555    fn method_url_uses_anchor() {
1556        let segs = vec!["runtime".into(), "Runtime".into(), "new".into()];
1557        let u = get_item_url("tokio", "latest", ItemKind::Fn, &segs).unwrap();
1558        assert!(
1559            u.as_str().contains("struct.Runtime.html#method.new"),
1560            "url={u}"
1561        );
1562    }
1563
1564    #[test]
1565    fn stdlib_resolved_version_is_channel() {
1566        let u = Url::parse("https://doc.rust-lang.org/stable/std/index.html").unwrap();
1567        assert_eq!(
1568            extract_resolved_version_for_crate(&u, "latest", Some("std")).as_deref(),
1569            Some("stable")
1570        );
1571    }
1572
1573    #[test]
1574    fn resolved_version_from_url() {
1575        let u = Url::parse("https://docs.rs/serde/1.0.210/serde/index.html").unwrap();
1576        assert_eq!(
1577            extract_resolved_version(&u, "latest").as_deref(),
1578            Some("1.0.210")
1579        );
1580        let u2 = Url::parse("https://docs.rs/serde/latest/serde/index.html").unwrap();
1581        assert!(extract_resolved_version(&u2, "latest").is_none());
1582    }
1583
1584    #[test]
1585    fn scrape_version_prefers_target_crate_not_deps() {
1586        let html = r#"
1587        <title>tokio - Rust</title>
1588        <a href="https://docs.rs/bytes/1.2.0/bytes/">bytes</a>
1589        <a href="/tokio/1.53.0/tokio/runtime/index.html">runtime</a>
1590        <a href="https://docs.rs/mio/0.8.0/mio/">mio</a>
1591        "#;
1592        assert_eq!(
1593            scrape_docs_rs_version_from_html(html, "tokio").as_deref(),
1594            Some("1.53.0")
1595        );
1596        // Must not pick dependency when crate path missing
1597        let html2 = r#"<title>x</title><a href="https://docs.rs/bytes/1.2.0/bytes/">b</a>"#;
1598        assert_eq!(scrape_docs_rs_version_from_html(html2, "tokio"), None);
1599    }
1600
1601    #[test]
1602    fn module_url_template() {
1603        let segs = vec!["serde".into(), "de".into()];
1604        let u = get_item_url("serde", "latest", ItemKind::Module, &segs).unwrap();
1605        assert!(u.as_str().ends_with("/serde/de/index.html"));
1606    }
1607
1608    #[test]
1609    fn empty_html_readme() {
1610        let (md, empty) = extract_readme_markdown_from_html("<html><body></body></html>").unwrap();
1611        assert!(empty);
1612        assert!(md.is_empty());
1613    }
1614
1615    #[test]
1616    fn get_item_empty_segments_errors() {
1617        let err = get_item_url("clap", "latest", ItemKind::Struct, &[]).unwrap_err();
1618        assert_eq!(err.kind(), ErrorKind::InvalidInput);
1619    }
1620
1621    #[test]
1622    fn join_full_http_url_passthrough() {
1623        let base = Url::parse("https://docs.rs/serde/latest/serde/").unwrap();
1624        let full = "https://docs.rs/serde/latest/serde/struct.Error.html";
1625        assert_eq!(join_href(&base, full).unwrap(), full);
1626        let http = "http://example.invalid/x.html";
1627        assert_eq!(join_href(&base, http).unwrap(), http);
1628    }
1629
1630    #[test]
1631    fn origin_builders_trim_trailing_slash() {
1632        let u = readme_url_on_origin("https://docs.rs/", "demo", "1.0.0").unwrap();
1633        assert_eq!(u.as_str(), "https://docs.rs/demo/1.0.0/demo/index.html");
1634        let a = all_html_url_on_origin("https://docs.rs/", "demo", "1.0.0").unwrap();
1635        assert_eq!(a.as_str(), "https://docs.rs/demo/1.0.0/demo/all.html");
1636    }
1637
1638    #[test]
1639    fn empty_item_html_extract() {
1640        let (md, empty) = extract_item_markdown_from_html("<html><body></body></html>").unwrap();
1641        assert!(empty);
1642        assert!(md.is_empty());
1643    }
1644
1645    #[test]
1646    fn method_extract_scopes_to_details_not_full_page() {
1647        let html = include_str!("../tests/fixtures/docs_rs/method_runtime_new.html");
1648        let (md, empty, scope) = extract_method_markdown_scoped(html, "new").unwrap();
1649        assert!(!empty);
1650        assert_eq!(scope, "method");
1651        assert!(
1652            !md.contains("parent page noise"),
1653            "must not dump full parent page: {md}"
1654        );
1655        assert!(
1656            md.to_ascii_lowercase().contains("new") || md.contains("Creates a new runtime"),
1657            "expected method content: {md}"
1658        );
1659        assert!(!md.contains('§'));
1660        assert!(!md.contains("Copy item path"));
1661    }
1662
1663    #[test]
1664    fn scrub_rustdoc_chrome_strips_section_sign() {
1665        let dirty = "## [§](#serde)Serde\nCopy item path\nbody";
1666        let clean = scrub_rustdoc_chrome(dirty);
1667        assert!(!clean.contains('§'));
1668        assert!(!clean.contains("Copy item path"));
1669        assert!(clean.contains("Serde"));
1670    }
1671
1672    #[test]
1673    fn method_missing_anchor_falls_back_item_page() {
1674        let html = r#"<html><body><div id="main-content"><h1>Struct Runtime</h1><p>only parent</p></div></body></html>"#;
1675        let (md, empty, scope) = extract_method_markdown_scoped(html, "new").unwrap();
1676        assert_eq!(scope, "item_page");
1677        assert!(!empty);
1678        assert!(md.contains("Runtime") || md.contains("parent"));
1679    }
1680}