Skip to main content

docsrs_cli/
docs_rs.rs

1//! docs.rs operations: readme, get-item, search-in-crate.
2//!
3//! # Extraction posture (Rules Rust — web scraping / data extraction)
4//!
5//! - **Transport** is HTTP GET ([`HttpClient`]); this module only parses bytes.
6//! - **Structure** uses `scraper` (html5ever) + CSS selectors — never regex for
7//!   field extraction from HTML.
8//! - **Sanitization** may use process-static regex only to strip `on*` handlers
9//!   and `javascript:` URLs after DOM script/style removal (XSS hygiene, not
10//!   content discovery).
11//! - **Markdown** conversion via `htmd` on the sanitized fragment.
12//! - **Provenance:** every success payload includes `source_url` (final URL).
13//! - **Encoding:** UTF-8 only after BOM strip ([`crate::http::decode_utf8`]);
14//!   docs.rs / doc.rust-lang.org ship UTF-8. `encoding_rs` multi-charset is OOS.
15//! - **Not a crawl:** no link frontier, no sitemap, no RSS/Atom, no robots meta
16//!   index policy (agent one-shot read, not dataset indexing).
17//!
18//! # Workload classification (Rules Rust — parallelism)
19//!
20//! - **I/O stage:** HTTPS GET via [`HttpClient`] on multi-thread Tokio workers.
21//! - **CPU stage:** HTML scrape + markdown conversion via
22//!   [`ConcurrencyBudget::run_cpu_bound`] (`spawn_blocking` + Semaphore).
23//! - **Hit scan:** large `all.html` candidate lists use `rayon` after a size
24//!   threshold; small lists stay sequential to avoid pool overhead.
25
26use std::collections::HashSet;
27use std::sync::LazyLock;
28
29use rayon::prelude::*;
30use regex::Regex;
31use scraper::{Html, Selector};
32use serde::{Deserialize, Serialize};
33use url::Url;
34
35use crate::config::{
36    HOST_DOC_RUST_LANG_ORG, HOST_DOCS_RS, MAX_SEARCH_IN_CRATE_LIMIT, is_stdlib_crate,
37    stdlib_channel,
38};
39use crate::error::{AppError, AppResult, ErrorKind};
40use crate::http::{HttpClient, content_type_looks_html, decode_utf8};
41use crate::item_kind::{ItemKind, rustc_crate_name};
42
43/// Crate overview (rustdoc crate docblock, not git README).
44#[derive(Debug, Clone, Serialize, Deserialize)]
45pub struct ReadmeData {
46    /// Requested crate name.
47    pub crate_name: String,
48    /// Requested version token.
49    pub version: String,
50    /// Version resolved by docs.rs redirects when known.
51    #[serde(skip_serializing_if = "Option::is_none")]
52    pub resolved_version: Option<String>,
53    /// Markdown extracted from the crate overview.
54    pub markdown: String,
55    /// True when the overview body is empty.
56    pub empty: bool,
57    /// True when output was truncated by `max_output_bytes`.
58    pub truncated: bool,
59    /// Final source URL of the HTML page.
60    pub source_url: String,
61}
62
63/// Typed item documentation payload.
64#[derive(Debug, Clone, Serialize, Deserialize)]
65pub struct GetItemData {
66    /// Requested crate name.
67    pub crate_name: String,
68    /// Normalized item kind string.
69    pub item_type: String,
70    /// Requested item path.
71    pub item_path: String,
72    /// Requested version token.
73    pub version: String,
74    /// Version resolved by docs.rs redirects when known.
75    #[serde(skip_serializing_if = "Option::is_none")]
76    pub resolved_version: Option<String>,
77    /// Markdown extracted from the item page.
78    pub markdown: String,
79    /// True when the item body is empty.
80    pub empty: bool,
81    /// True when output was truncated by `max_output_bytes`.
82    pub truncated: bool,
83    /// Final source URL of the HTML page.
84    pub source_url: String,
85    /// Page title when available.
86    pub title: String,
87}
88
89/// Single all.html hit.
90#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
91pub struct SearchInCrateHit {
92    /// Symbol or path name from all.html.
93    pub name: String,
94    /// Item kind string (`struct`, `fn`, …).
95    pub kind: String,
96    /// Absolute URL of the item documentation page.
97    pub url: String,
98}
99
100/// search-in-crate result set (`truncated` is true when `total > emitted`).
101#[derive(Debug, Clone, Serialize, Deserialize)]
102pub struct SearchInCrateData {
103    /// Requested crate name.
104    pub crate_name: String,
105    /// Substring filter applied to hits.
106    pub query: String,
107    /// Requested version token.
108    pub version: String,
109    /// Total classified hits before the limit.
110    pub total: usize,
111    /// Number of hits actually emitted.
112    pub emitted: usize,
113    /// Emitted hits.
114    pub hits: Vec<SearchInCrateHit>,
115    /// True when the hit list was cut by `--limit` (`total > emitted`).
116    pub truncated: bool,
117    /// Final source URL of the all.html page.
118    pub source_url: String,
119}
120
121fn default_docs_origin(crate_name: &str) -> String {
122    if is_stdlib_crate(crate_name) {
123        format!("https://{HOST_DOC_RUST_LANG_ORG}")
124    } else {
125        format!("https://{HOST_DOCS_RS}")
126    }
127}
128
129/// Build crate index URL with rustc hyphen→underscore segment.
130///
131/// # Errors
132///
133/// Propagates [`ErrorKind::Internal`] from [`readme_url_on_origin`] when the URL is invalid.
134pub fn readme_url(crate_name: &str, version: &str) -> AppResult<Url> {
135    readme_url_on_origin(&default_docs_origin(crate_name), crate_name, version)
136}
137
138/// Build readme URL against a custom origin (wiremock tests).
139///
140/// # Errors
141///
142/// Returns [`ErrorKind::Internal`] when `origin` / path segments do not form a valid URL.
143pub fn readme_url_on_origin(origin: &str, crate_name: &str, version: &str) -> AppResult<Url> {
144    let origin = origin.trim_end_matches('/');
145    let s = if is_stdlib_crate(crate_name) {
146        // doc.rust-lang.org/{channel}/{crate}/index.html
147        let channel = stdlib_channel(version);
148        format!("{origin}/{channel}/{crate_name}/index.html")
149    } else {
150        let rustc = rustc_crate_name(crate_name);
151        format!("{origin}/{crate_name}/{version}/{rustc}/index.html")
152    };
153    Url::parse(&s).map_err(|e| AppError::with_source(ErrorKind::Internal, "invalid readme URL", e))
154}
155
156/// Build rustdoc item or module URL.
157///
158/// # Errors
159///
160/// Propagates [`ErrorKind::InvalidInput`] or [`ErrorKind::Internal`] from
161/// [`get_item_url_on_origin`].
162pub fn get_item_url(
163    crate_name: &str,
164    version: &str,
165    kind: ItemKind,
166    segments: &[String],
167) -> AppResult<Url> {
168    get_item_url_on_origin(
169        &default_docs_origin(crate_name),
170        crate_name,
171        version,
172        kind,
173        segments,
174    )
175}
176
177/// Build get-item URL against a custom origin (wiremock tests).
178///
179/// # Errors
180///
181/// Returns [`ErrorKind::InvalidInput`] when a non-module path lacks an item name.
182/// Returns [`ErrorKind::Internal`] when the assembled path is not a valid URL.
183pub fn get_item_url_on_origin(
184    origin: &str,
185    crate_name: &str,
186    version: &str,
187    kind: ItemKind,
188    segments: &[String],
189) -> AppResult<Url> {
190    let origin = origin.trim_end_matches('/');
191    let rustc_root = rustc_crate_name(crate_name);
192    // Optional leading crate prefix (`tokio::runtime::Runtime`) is stripped only when
193    // it is a path prefix (2+ segments) or the target is the crate-root module.
194    // A single segment equal to the crate name is the item itself (e.g. attribute
195    // `async_trait` on crate `async-trait`) and must NOT be stripped.
196    let segs: Vec<String> = {
197        let mut s = segments.to_vec();
198        if let Some(first) = s.first() {
199            let f = first.as_str();
200            let is_crate_prefix = f == crate_name || f == rustc_root.as_str();
201            if is_crate_prefix && (s.len() >= 2 || kind == ItemKind::Module) {
202                s.remove(0);
203            }
204        }
205        s
206    };
207
208    let url_str = if is_stdlib_crate(crate_name) {
209        // doc.rust-lang.org/{channel}/{crate}/[mod/]{kind}.{Name}.html
210        let channel = stdlib_channel(version);
211        if kind == ItemKind::Module {
212            let mut parts: Vec<String> = Vec::new();
213            for p in &segs {
214                parts.push(rustc_crate_name(p));
215            }
216            if parts.is_empty() {
217                format!("{origin}/{channel}/{crate_name}/index.html")
218            } else {
219                format!(
220                    "{origin}/{channel}/{crate_name}/{}/index.html",
221                    parts.join("/")
222                )
223            }
224        } else {
225            if segs.is_empty() {
226                return Err(AppError::new(
227                    ErrorKind::InvalidInput,
228                    "item path missing item name",
229                ));
230            }
231            let item_name = segs.last().ok_or_else(|| {
232                AppError::new(ErrorKind::InvalidInput, "item path missing item name")
233            })?;
234            let mod_parts: Vec<String> = segs[..segs.len().saturating_sub(1)]
235                .iter()
236                .map(|p| rustc_crate_name(p))
237                .collect();
238            if mod_parts.is_empty() {
239                format!(
240                    "{origin}/{channel}/{crate_name}/{}.{}.html",
241                    kind.file_prefix(),
242                    item_name
243                )
244            } else {
245                format!(
246                    "{origin}/{channel}/{crate_name}/{}/{}.{}.html",
247                    mod_parts.join("/"),
248                    kind.file_prefix(),
249                    item_name
250                )
251            }
252        }
253    } else if kind == ItemKind::Module {
254        let mut parts: Vec<String> = vec![rustc_root];
255        for p in &segs {
256            parts.push(rustc_crate_name(p));
257        }
258        format!(
259            "{origin}/{crate_name}/{version}/{}/index.html",
260            parts.join("/")
261        )
262    } else {
263        if segs.is_empty() {
264            return Err(AppError::new(
265                ErrorKind::InvalidInput,
266                "item path missing item name",
267            ));
268        }
269        let item_name = segs
270            .last()
271            .ok_or_else(|| AppError::new(ErrorKind::InvalidInput, "item path missing item name"))?;
272        let mod_parts: Vec<String> = if segs.len() == 1 {
273            vec![rustc_root]
274        } else {
275            let mut m = vec![rustc_root];
276            for p in &segs[..segs.len() - 1] {
277                m.push(rustc_crate_name(p));
278            }
279            m
280        };
281        format!(
282            "{origin}/{crate_name}/{version}/{}/{}.{}.html",
283            mod_parts.join("/"),
284            kind.file_prefix(),
285            item_name
286        )
287    };
288
289    Url::parse(&url_str)
290        .map_err(|e| AppError::with_source(ErrorKind::Internal, "invalid get-item URL", e))
291}
292
293/// Build all.html index URL.
294///
295/// # Errors
296///
297/// Propagates [`ErrorKind::Internal`] from [`all_html_url_on_origin`].
298pub fn all_html_url(crate_name: &str, version: &str) -> AppResult<Url> {
299    all_html_url_on_origin(&default_docs_origin(crate_name), crate_name, version)
300}
301
302/// Build all.html URL against a custom origin (wiremock tests).
303///
304/// # Errors
305///
306/// Returns [`ErrorKind::Internal`] when `origin` / path segments do not form a valid URL.
307pub fn all_html_url_on_origin(origin: &str, crate_name: &str, version: &str) -> AppResult<Url> {
308    let origin = origin.trim_end_matches('/');
309    let s = if is_stdlib_crate(crate_name) {
310        let channel = stdlib_channel(version);
311        format!("{origin}/{channel}/{crate_name}/all.html")
312    } else {
313        let rustc = rustc_crate_name(crate_name);
314        format!("{origin}/{crate_name}/{version}/{rustc}/all.html")
315    };
316    Url::parse(&s)
317        .map_err(|e| AppError::with_source(ErrorKind::Internal, "invalid all.html URL", e))
318}
319
320fn extract_resolved_version(final_url: &Url, requested: &str) -> Option<String> {
321    let mut segs = final_url.path_segments()?.filter(|s| !s.is_empty());
322    let _pkg = segs.next()?;
323    let ver = segs.next()?;
324    if ver == "latest" {
325        return None;
326    }
327    if requested == "latest" || ver != requested {
328        return Some(ver.to_string());
329    }
330    Some(ver.to_string())
331}
332
333/// Compiled once: strip `on*` event handlers from HTML fragments (hot path for every HTML→MD).
334static RE_ON_HANDLERS: LazyLock<Regex> = LazyLock::new(|| {
335    Regex::new(r#"(?i)\s+on[a-z]+\s*=\s*("[^"]*"|'[^']*'|[^\s>]+)"#)
336        .expect("hardcoded on* handler strip regex is valid by construction")
337});
338
339/// Compiled once: strip `javascript:` URLs from href/src attributes.
340static RE_JS_URLS: LazyLock<Regex> = LazyLock::new(|| {
341    Regex::new(r#"(?i)(href|src)\s*=\s*["']\s*javascript:[^"']*["']"#)
342        .expect("hardcoded javascript: URL strip regex is valid by construction")
343});
344
345/// Remove script/style nodes and on* handlers before Markdown conversion.
346///
347/// Extraction of doc content uses CSS selectors elsewhere in this module.
348/// Regexes here are **security scrub only** (`on*` / `javascript:`), compiled
349/// once via process-static [`LazyLock`] — not used to scrape fields from HTML.
350pub fn sanitize_html_fragment(html: &str) -> String {
351    let fragment = Html::parse_fragment(html);
352    let drop_sel = Selector::parse("script, style, noscript")
353        .expect("hardcoded scraper selector 'script, style, noscript' is valid by construction");
354    // Prefer reconstructing from root children excluding dropped tags.
355    if let Some(root) = fragment.tree.root().children().next() {
356        let _ = root;
357    }
358    // Collect HTML of non-script/style nodes by stripping matches from a working copy.
359    // Capacity ≈ input length: script/style removals only shrink the buffer.
360    // Precondition: `html` is already body-capped (HARD_MAX_BODY_BYTES) and resident in RAM.
361    let mut out = String::with_capacity(html.len());
362    out.push_str(html);
363    for el in fragment.select(&drop_sel) {
364        let tag_html = el.html();
365        out = out.replace(&tag_html, "");
366    }
367    // Strip event handlers: onload=, onclick=, etc.
368    out = RE_ON_HANDLERS.replace_all(&out, "").into_owned();
369    // Drop javascript: URLs in href/src
370    out = RE_JS_URLS.replace_all(&out, "").into_owned();
371    out
372}
373
374fn html_to_markdown(html: &str) -> AppResult<String> {
375    let cleaned = sanitize_html_fragment(html);
376    htmd::convert(&cleaned).map_err(|e| {
377        AppError::new(
378            ErrorKind::Parse,
379            format!("HTML to Markdown conversion failed: {e}"),
380        )
381    })
382}
383
384fn first_inner_html(document: &Html, selectors: &[&str]) -> Option<String> {
385    for sel in selectors {
386        if let Ok(s) = Selector::parse(sel)
387            && let Some(n) = document.select(&s).next()
388        {
389            let inner = n.inner_html();
390            if !inner.trim().is_empty() {
391                return Some(inner);
392            }
393        }
394    }
395    None
396}
397
398/// Fetch and convert crate overview docblock (production docs.rs).
399///
400/// # Errors
401///
402/// Propagates URL, HTTP, decode, and HTML conversion failures from
403/// [`fetch_readme_on_origin`].
404pub async fn fetch_readme(
405    http: &HttpClient,
406    crate_name: &str,
407    version: &str,
408) -> AppResult<ReadmeData> {
409    fetch_readme_on_origin(
410        http,
411        &format!("https://{HOST_DOCS_RS}"),
412        crate_name,
413        version,
414    )
415    .await
416}
417
418/// Fetch readme against a configurable origin (offline mocks).
419///
420/// # Errors
421///
422/// Returns [`ErrorKind::Internal`] when the origin URL is invalid.
423/// Propagates fetch failures from [`fetch_readme_at`].
424pub async fn fetch_readme_on_origin(
425    http: &HttpClient,
426    origin: &str,
427    crate_name: &str,
428    version: &str,
429) -> AppResult<ReadmeData> {
430    let url = readme_url_on_origin(origin, crate_name, version)?;
431    fetch_readme_at(http, crate_name, version, &url).await
432}
433
434/// Fetch readme from a prebuilt URL (wiremock).
435///
436/// # Errors
437///
438/// Propagates HTTP errors from [`HttpClient::get_html`].
439/// Maps non-success statuses via [`AppError::from_http_status`].
440/// Returns [`ErrorKind::Parse`] on non-UTF-8 bodies or HTML→Markdown failure.
441pub async fn fetch_readme_at(
442    http: &HttpClient,
443    crate_name: &str,
444    version: &str,
445    url: &Url,
446) -> AppResult<ReadmeData> {
447    let resp = http.get_html(url).await?;
448    if resp.status.as_u16() == 404 {
449        return Err(AppError::from_http_status(
450            404,
451            &format!("crate or version not found: {crate_name}@{version}"),
452        ));
453    }
454    if !resp.status.is_success() {
455        return Err(AppError::from_http_status(
456            resp.status.as_u16(),
457            "docs.rs readme",
458        ));
459    }
460    if resp.content_type.is_some() && !content_type_looks_html(resp.content_type.as_deref()) {
461        tracing::warn!(content_type = ?resp.content_type, "unexpected Content-Type from docs.rs");
462    }
463
464    let body = resp.body.clone();
465    let (markdown, empty) = http
466        .budget()
467        .run_cpu_bound(move || {
468            let text = decode_utf8(&body)?;
469            extract_readme_markdown_from_html(&text)
470        })
471        .await?;
472    let resolved = extract_resolved_version(&resp.final_url, version);
473    Ok(ReadmeData {
474        crate_name: crate_name.to_string(),
475        version: version.to_string(),
476        resolved_version: resolved,
477        markdown,
478        empty,
479        truncated: false,
480        source_url: resp.final_url.to_string(),
481    })
482}
483
484/// Fetch and convert a typed rustdoc item page (production docs.rs).
485///
486/// # Errors
487///
488/// Propagates URL, HTTP, decode, and HTML conversion failures from
489/// [`fetch_item_on_origin`].
490pub async fn fetch_item(
491    http: &HttpClient,
492    crate_name: &str,
493    version: &str,
494    kind: ItemKind,
495    segments: &[String],
496) -> AppResult<GetItemData> {
497    fetch_item_on_origin(
498        http,
499        &format!("https://{HOST_DOCS_RS}"),
500        crate_name,
501        version,
502        kind,
503        segments,
504    )
505    .await
506}
507
508/// Fetch item against a configurable origin (offline mocks).
509///
510/// # Errors
511///
512/// Propagates URL build errors from [`get_item_url_on_origin`] and fetch errors
513/// from [`fetch_item_at`].
514pub async fn fetch_item_on_origin(
515    http: &HttpClient,
516    origin: &str,
517    crate_name: &str,
518    version: &str,
519    kind: ItemKind,
520    segments: &[String],
521) -> AppResult<GetItemData> {
522    let url = get_item_url_on_origin(origin, crate_name, version, kind, segments)?;
523    fetch_item_at(http, crate_name, version, kind, segments, &url).await
524}
525
526/// Fetch item from a prebuilt URL (wiremock).
527///
528/// # Errors
529///
530/// Propagates HTTP errors from [`HttpClient::get_html`].
531/// Maps non-success statuses via [`AppError::from_http_status`].
532/// Returns [`ErrorKind::Parse`] on non-UTF-8 bodies or HTML→Markdown failure.
533pub async fn fetch_item_at(
534    http: &HttpClient,
535    crate_name: &str,
536    version: &str,
537    kind: ItemKind,
538    segments: &[String],
539    url: &Url,
540) -> AppResult<GetItemData> {
541    let resp = http.get_html(url).await?;
542    if resp.status.as_u16() == 404 {
543        return Err(AppError::from_http_status(
544            404,
545            &format!("item not found at {url}"),
546        ));
547    }
548    if !resp.status.is_success() {
549        return Err(AppError::from_http_status(
550            resp.status.as_u16(),
551            "docs.rs get-item",
552        ));
553    }
554
555    let body = resp.body.clone();
556    let (markdown, empty) = http
557        .budget()
558        .run_cpu_bound(move || {
559            let text = decode_utf8(&body)?;
560            extract_item_markdown_from_html(&text)
561        })
562        .await?;
563    let item_path = segments.join("::");
564    let title = format!("{item_path} ({})", kind.as_str());
565    let resolved = extract_resolved_version(&resp.final_url, version);
566
567    Ok(GetItemData {
568        crate_name: crate_name.to_string(),
569        item_type: kind.as_str().to_string(),
570        item_path,
571        version: version.to_string(),
572        resolved_version: resolved,
573        markdown,
574        empty,
575        truncated: false,
576        source_url: resp.final_url.to_string(),
577        title,
578    })
579}
580
581/// Parse all.html and filter symbols (production docs.rs).
582///
583/// # Errors
584///
585/// Propagates URL, HTTP, decode, and parse failures from
586/// [`search_in_crate_on_origin`].
587pub async fn search_in_crate(
588    http: &HttpClient,
589    crate_name: &str,
590    version: &str,
591    query: &str,
592    item_type: Option<ItemKind>,
593    limit: u32,
594) -> AppResult<SearchInCrateData> {
595    search_in_crate_on_origin(
596        http,
597        &format!("https://{HOST_DOCS_RS}"),
598        crate_name,
599        version,
600        query,
601        item_type,
602        limit,
603    )
604    .await
605}
606
607/// search-in-crate against a configurable origin (offline mocks).
608///
609/// # Errors
610///
611/// Propagates URL build errors from [`all_html_url_on_origin`] and fetch errors
612/// from [`search_in_crate_at`].
613pub async fn search_in_crate_on_origin(
614    http: &HttpClient,
615    origin: &str,
616    crate_name: &str,
617    version: &str,
618    query: &str,
619    item_type: Option<ItemKind>,
620    limit: u32,
621) -> AppResult<SearchInCrateData> {
622    let url = all_html_url_on_origin(origin, crate_name, version)?;
623    search_in_crate_at(http, crate_name, version, query, item_type, limit, &url).await
624}
625
626/// search-in-crate against a prebuilt all.html URL (wiremock).
627///
628/// # Errors
629///
630/// Propagates HTTP errors from [`HttpClient::get_html`].
631/// Maps non-success statuses via [`AppError::from_http_status`].
632/// Returns [`ErrorKind::Parse`] on non-UTF-8 bodies or href join failures.
633/// Propagates parse errors from [`search_in_crate_from_html`].
634pub async fn search_in_crate_at(
635    http: &HttpClient,
636    crate_name: &str,
637    version: &str,
638    query: &str,
639    item_type: Option<ItemKind>,
640    limit: u32,
641    url: &Url,
642) -> AppResult<SearchInCrateData> {
643    let resp = http.get_html(url).await?;
644    if resp.status.as_u16() == 404 {
645        return Err(AppError::from_http_status(
646            404,
647            &format!("all.html not found for {crate_name}@{version}"),
648        ));
649    }
650    if !resp.status.is_success() {
651        return Err(AppError::from_http_status(
652            resp.status.as_u16(),
653            "docs.rs all.html",
654        ));
655    }
656
657    let body = resp.body.clone();
658    let source_url = resp.final_url.to_string();
659    let crate_name = crate_name.to_string();
660    let version = version.to_string();
661    let query = query.to_string();
662    http.budget()
663        .run_cpu_bound(move || {
664            let text = decode_utf8(&body)?;
665            search_in_crate_from_html(
666                &text,
667                &crate_name,
668                &version,
669                &query,
670                item_type,
671                limit,
672                &source_url,
673            )
674        })
675        .await
676}
677
678/// Join relative/absolute/full href against crate rustdoc base.
679///
680/// # Errors
681///
682/// Returns [`ErrorKind::Parse`] when a relative or absolute-path href cannot be joined.
683pub fn join_href(base: &Url, href: &str) -> AppResult<String> {
684    if href.starts_with("http://") || href.starts_with("https://") {
685        return Ok(href.to_string());
686    }
687    if href.starts_with('/') {
688        let joined = Url::parse(&format!("https://docs.rs{href}")).map_err(|e| {
689            AppError::with_source(ErrorKind::Parse, "failed to join absolute path href", e)
690        })?;
691        return Ok(joined.to_string());
692    }
693    base.join(href)
694        .map(|u| u.to_string())
695        .map_err(|e| AppError::with_source(ErrorKind::Parse, "failed to join relative href", e))
696}
697
698/// Minimum candidate count before `rayon` fan-out (overhead below this is pure loss).
699const RAYON_HIT_THRESHOLD: usize = 64;
700
701/// One classified hit candidate before dedup/limit (index preserves document order).
702type ClassifiedHit = (usize, String, ItemKind, String);
703
704/// Pure parse of all.html body for offline tests and CPU workers.
705///
706/// Large candidate lists use `rayon`; small lists stay sequential.
707///
708/// # Errors
709///
710/// Returns [`ErrorKind::Internal`] when the synthetic base URL is invalid.
711/// Returns [`ErrorKind::Parse`] when an href cannot be joined via [`join_href`].
712pub fn parse_all_html_hits(
713    html: &str,
714    crate_name: &str,
715    version: &str,
716    query: &str,
717    item_type: Option<ItemKind>,
718    limit: usize,
719) -> AppResult<Vec<SearchInCrateHit>> {
720    if limit == 0 {
721        return Ok(Vec::new());
722    }
723    let document = Html::parse_document(html);
724    let a_sel = Selector::parse("#main-content a")
725        .expect("hardcoded scraper selector '#main-content a' is valid by construction");
726    let rustc = rustc_crate_name(crate_name);
727    let base = Url::parse(&format!(
728        "https://{HOST_DOCS_RS}/{crate_name}/{version}/{rustc}/"
729    ))
730    .map_err(|e| AppError::with_source(ErrorKind::Internal, "invalid all.html base", e))?;
731    let q = query.trim().to_ascii_lowercase();
732
733    // Materialize (name, href) so we can choose sequential vs rayon without holding DOM refs.
734    let candidates: Vec<(String, String)> = document
735        .select(&a_sel)
736        .filter_map(|a| {
737            let name = a.text().collect::<String>().trim().to_string();
738            if name.is_empty() {
739                return None;
740            }
741            let href = a.value().attr("href")?.to_string();
742            if href.is_empty() {
743                return None;
744            }
745            Some((name, href))
746        })
747        .collect();
748
749    if candidates.len() < RAYON_HIT_THRESHOLD {
750        return filter_hits_sequential(candidates, &base, &q, item_type, limit);
751    }
752    filter_hits_parallel(candidates, &base, &q, item_type, limit)
753}
754
755fn filter_hits_sequential(
756    candidates: Vec<(String, String)>,
757    base: &Url,
758    q: &str,
759    item_type: Option<ItemKind>,
760    limit: usize,
761) -> AppResult<Vec<SearchInCrateHit>> {
762    let mut hits = Vec::with_capacity(limit.min(256));
763    let mut seen: HashSet<(String, ItemKind)> = HashSet::new();
764    for (name, href) in candidates {
765        if let Some(hit) = classify_hit(name, &href, base, q, item_type, &mut seen)? {
766            hits.push(hit);
767            if hits.len() >= limit {
768                break;
769            }
770        }
771    }
772    Ok(hits)
773}
774
775fn filter_hits_parallel(
776    candidates: Vec<(String, String)>,
777    base: &Url,
778    q: &str,
779    item_type: Option<ItemKind>,
780    limit: usize,
781) -> AppResult<Vec<SearchInCrateHit>> {
782    // Parallel classify/filter; preserve original order via index for stable agent output.
783    let mapped: AppResult<Vec<Option<ClassifiedHit>>> = candidates
784        .into_par_iter()
785        .enumerate()
786        .map(|(idx, (name, href))| {
787            let Some(kind) = ItemKind::from_href(&href) else {
788                return Ok(None);
789            };
790            if let Some(filter) = item_type
791                && kind != filter
792            {
793                return Ok(None);
794            }
795            if !q.is_empty() && !name.to_ascii_lowercase().contains(q) {
796                return Ok(None);
797            }
798            let abs = join_href(base, &href)?;
799            Ok(Some((idx, name, kind, abs)))
800        })
801        .collect();
802    let mut rows: Vec<ClassifiedHit> = mapped?.into_iter().flatten().collect();
803    rows.par_sort_unstable_by_key(|(idx, _, _, _)| *idx);
804
805    let mut hits = Vec::with_capacity(limit.min(256));
806    let mut seen: HashSet<(String, ItemKind)> = HashSet::new();
807    for (_idx, name, kind, abs) in rows {
808        if !seen.insert((name.clone(), kind)) {
809            continue;
810        }
811        hits.push(SearchInCrateHit {
812            name,
813            kind: kind.as_str().to_string(),
814            url: abs,
815        });
816        if hits.len() >= limit {
817            break;
818        }
819    }
820    Ok(hits)
821}
822
823fn classify_hit(
824    name: String,
825    href: &str,
826    base: &Url,
827    q: &str,
828    item_type: Option<ItemKind>,
829    seen: &mut HashSet<(String, ItemKind)>,
830) -> AppResult<Option<SearchInCrateHit>> {
831    let Some(kind) = ItemKind::from_href(href) else {
832        return Ok(None);
833    };
834    if let Some(filter) = item_type
835        && kind != filter
836    {
837        return Ok(None);
838    }
839    if !q.is_empty() && !name.to_ascii_lowercase().contains(q) {
840        return Ok(None);
841    }
842    let abs = join_href(base, href)?;
843    if !seen.insert((name.clone(), kind)) {
844        return Ok(None);
845    }
846    Ok(Some(SearchInCrateHit {
847        name,
848        kind: kind.as_str().to_string(),
849        url: abs,
850    }))
851}
852
853/// Extract readme markdown from raw HTML (offline tests / pure path).
854///
855/// # Errors
856///
857/// Returns [`ErrorKind::Parse`] when HTML→Markdown conversion fails.
858pub fn extract_readme_markdown_from_html(html: &str) -> AppResult<(String, bool)> {
859    let document = Html::parse_document(html);
860    let content = first_inner_html(
861        &document,
862        &[
863            ".rustdoc .docblock",
864            "#main-content .docblock",
865            ".docblock",
866            ".rustdoc-main .item-decl",
867            ".item-decl",
868        ],
869    );
870    match content {
871        Some(h) => Ok((html_to_markdown(&h)?, false)),
872        None => Ok((String::new(), true)),
873    }
874}
875
876/// Extract get-item markdown from raw HTML (offline tests / pure path).
877///
878/// # Errors
879///
880/// Returns [`ErrorKind::Parse`] when HTML→Markdown conversion fails.
881pub fn extract_item_markdown_from_html(html: &str) -> AppResult<(String, bool)> {
882    let document = Html::parse_document(html);
883    let content = first_inner_html(&document, &["#main-content", ".item-decl", ".docblock"])
884        .unwrap_or_default();
885    let empty = content.trim().is_empty();
886    if empty {
887        Ok((String::new(), true))
888    } else {
889        Ok((html_to_markdown(&content)?, false))
890    }
891}
892
893/// Build SearchInCrateData from HTML body without network (offline tests).
894///
895/// `--limit 0` is honoured: `emitted = 0`, `hits = []`, and `truncated` is true
896/// when the unfiltered total is greater than zero. Values above
897/// [`MAX_SEARCH_IN_CRATE_LIMIT`] are capped.
898///
899/// # Errors
900///
901/// Propagates parse failures from [`parse_all_html_hits`].
902pub fn search_in_crate_from_html(
903    html: &str,
904    crate_name: &str,
905    version: &str,
906    query: &str,
907    item_type: Option<ItemKind>,
908    limit: u32,
909    source_url: &str,
910) -> AppResult<SearchInCrateData> {
911    let limit = (limit as usize).min(MAX_SEARCH_IN_CRATE_LIMIT as usize);
912    let all = parse_all_html_hits(html, crate_name, version, query, item_type, usize::MAX)?;
913    let total = all.len();
914    let hits: Vec<_> = all.into_iter().take(limit).collect();
915    let emitted = hits.len();
916    Ok(SearchInCrateData {
917        crate_name: crate_name.to_string(),
918        query: query.to_string(),
919        version: version.to_string(),
920        total,
921        emitted,
922        hits,
923        truncated: total > emitted,
924        source_url: source_url.to_string(),
925    })
926}
927
928#[cfg(test)]
929mod tests {
930    use super::*;
931
932    #[test]
933    fn readme_url_hyphen() {
934        let u = readme_url("async-trait", "latest").unwrap();
935        assert_eq!(
936            u.as_str(),
937            "https://docs.rs/async-trait/latest/async_trait/index.html"
938        );
939    }
940
941    #[test]
942    fn get_item_nested() {
943        let segs = vec!["tokio".into(), "runtime".into(), "Runtime".into()];
944        let u = get_item_url("tokio", "latest", ItemKind::Struct, &segs).unwrap();
945        assert!(u.as_str().contains("/tokio/runtime/struct.Runtime.html"));
946    }
947
948    #[test]
949    fn get_item_without_crate_prefix() {
950        let segs = vec!["Parser".into()];
951        let u = get_item_url("clap", "latest", ItemKind::Trait, &segs).unwrap();
952        assert!(
953            u.as_str().ends_with("/clap/trait.Parser.html")
954                || u.path().ends_with("/clap/trait.Parser.html")
955        );
956    }
957
958    #[test]
959    fn get_item_keeps_single_segment_equal_to_crate_name() {
960        // Attribute/macro items often share the rustc crate name.
961        let segs = vec!["async_trait".into()];
962        let u = get_item_url("async-trait", "latest", ItemKind::Attribute, &segs).unwrap();
963        assert!(
964            u.as_str()
965                .ends_with("/async-trait/latest/async_trait/attr.async_trait.html"),
966            "url={u}"
967        );
968    }
969
970    #[test]
971    fn get_item_module_single_segment_crate_name_is_root() {
972        let segs = vec!["async_trait".into()];
973        let u = get_item_url("async-trait", "latest", ItemKind::Module, &segs).unwrap();
974        assert!(
975            u.as_str()
976                .ends_with("/async-trait/latest/async_trait/index.html"),
977            "url={u}"
978        );
979    }
980
981    #[test]
982    fn join_relative() {
983        let base = Url::parse("https://docs.rs/serde/latest/serde/").unwrap();
984        let j = join_href(&base, "de/trait.Deserialize.html").unwrap();
985        assert!(j.contains("/serde/de/trait.Deserialize.html"));
986    }
987
988    #[test]
989    fn join_absolute_path_and_fragment() {
990        let base = Url::parse("https://docs.rs/serde/latest/serde/").unwrap();
991        let j = join_href(&base, "/serde/latest/serde/struct.Error.html").unwrap();
992        assert!(j.starts_with("https://docs.rs/serde/"));
993        let f = join_href(&base, "struct.Error.html#method.to_string").unwrap();
994        assert!(f.contains("#method.to_string"));
995    }
996
997    #[test]
998    fn sanitize_strips_script() {
999        let raw = r#"<div>ok<script>alert(1)</script><p onclick="x()">hi</p></div>"#;
1000        let s = sanitize_html_fragment(raw);
1001        assert!(!s.contains("script"));
1002        assert!(!s.contains("onclick"));
1003        assert!(s.contains("hi") || s.contains("ok"));
1004    }
1005
1006    #[test]
1007    fn readme_fallback_item_decl_converts() {
1008        let html = r#"<!DOCTYPE html><html><body class="rustdoc-main">
1009        <div class="item-decl"><pre>pub use foo;</pre></div>
1010        </body></html>"#;
1011        let (md, empty) = extract_readme_markdown_from_html(html).unwrap();
1012        assert!(!empty);
1013        assert!(!md.trim().is_empty());
1014    }
1015
1016    #[test]
1017    fn all_html_order_stable() {
1018        let html = r#"<!DOCTYPE html><html><body>
1019        <div id="main-content">
1020          <a href="struct.A.html">A</a>
1021          <a href="trait.B.html">B</a>
1022          <a href="fn.c.html">c</a>
1023          <a href="union.U.html">U</a>
1024          <a href="attr.x.html">x</a>
1025          <a href="derive.D.html">D</a>
1026        </div></body></html>"#;
1027        let hits = parse_all_html_hits(html, "demo", "1.0.0", "", None, 100).unwrap();
1028        assert_eq!(hits.len(), 6);
1029        assert_eq!(hits[0].name, "A");
1030        assert_eq!(hits[0].kind, "struct");
1031        assert_eq!(hits[3].kind, "union");
1032        assert_eq!(hits[4].kind, "attribute");
1033        assert_eq!(hits[5].kind, "derive");
1034    }
1035
1036    #[test]
1037    fn std_core_alloc_urls() {
1038        let u = readme_url("std", "latest").unwrap();
1039        assert_eq!(
1040            u.as_str(),
1041            "https://doc.rust-lang.org/stable/std/index.html"
1042        );
1043        let u = get_item_url(
1044            "std",
1045            "latest",
1046            ItemKind::Struct,
1047            &["option".into(), "Option".into()],
1048        )
1049        .unwrap();
1050        assert_eq!(
1051            u.as_str(),
1052            "https://doc.rust-lang.org/stable/std/option/struct.Option.html"
1053        );
1054        let a = all_html_url("core", "nightly").unwrap();
1055        assert_eq!(
1056            a.as_str(),
1057            "https://doc.rust-lang.org/nightly/core/all.html"
1058        );
1059    }
1060
1061    #[test]
1062    fn extract_item_from_fixture() {
1063        let html = include_str!("../tests/fixtures/docs_rs/get_item_main.html");
1064        let (md, empty) = extract_item_markdown_from_html(html).unwrap();
1065        assert!(!empty);
1066        assert!(md.contains("Runtime") || md.contains("Tokio") || md.contains("runtime"));
1067    }
1068
1069    #[test]
1070    fn extract_readme_primary_docblock_fixture() {
1071        let html = include_str!("../tests/fixtures/docs_rs/readme_docblock.html");
1072        let (md, empty) = extract_readme_markdown_from_html(html).unwrap();
1073        assert!(!empty);
1074        assert!(!md.to_ascii_lowercase().contains("alert"));
1075    }
1076
1077    #[test]
1078    fn search_in_crate_from_html_limit_and_filter() {
1079        let html = include_str!("../tests/fixtures/docs_rs/all_html_sample.html");
1080        let data = search_in_crate_from_html(
1081            html,
1082            "demo",
1083            "1.0.0",
1084            "",
1085            Some(ItemKind::Struct),
1086            1,
1087            "https://docs.rs/demo/1.0.0/demo/all.html",
1088        )
1089        .unwrap();
1090        assert_eq!(data.total, 1);
1091        assert_eq!(data.emitted, 1);
1092        assert_eq!(data.hits[0].kind, "struct");
1093        assert!(!data.truncated);
1094    }
1095
1096    #[test]
1097    fn search_in_crate_truncated_when_limit_cuts_hits() {
1098        let html = include_str!("../tests/fixtures/docs_rs/all_html_sample.html");
1099        let data = search_in_crate_from_html(
1100            html,
1101            "demo",
1102            "1.0.0",
1103            "",
1104            None,
1105            2,
1106            "https://docs.rs/demo/1.0.0/demo/all.html",
1107        )
1108        .unwrap();
1109        assert!(data.total > 2, "fixture must have more than 2 hits");
1110        assert_eq!(data.emitted, 2);
1111        assert!(data.truncated);
1112        let full = search_in_crate_from_html(
1113            html,
1114            "demo",
1115            "1.0.0",
1116            "",
1117            None,
1118            1000,
1119            "https://docs.rs/demo/1.0.0/demo/all.html",
1120        )
1121        .unwrap();
1122        assert_eq!(full.total, full.emitted);
1123        assert!(!full.truncated);
1124    }
1125
1126    #[test]
1127    fn search_in_crate_limit_zero_emits_nothing() {
1128        let html = include_str!("../tests/fixtures/docs_rs/all_html_sample.html");
1129        let data = search_in_crate_from_html(
1130            html,
1131            "demo",
1132            "1.0.0",
1133            "",
1134            None,
1135            0,
1136            "https://docs.rs/demo/1.0.0/demo/all.html",
1137        )
1138        .unwrap();
1139        assert!(data.total > 0, "fixture must have hits for total");
1140        assert_eq!(data.emitted, 0);
1141        assert!(data.hits.is_empty());
1142        assert!(data.truncated);
1143        assert!(
1144            parse_all_html_hits(html, "demo", "1.0.0", "", None, 0)
1145                .unwrap()
1146                .is_empty()
1147        );
1148    }
1149
1150    #[test]
1151    fn resolved_version_from_url() {
1152        let u = Url::parse("https://docs.rs/serde/1.0.210/serde/index.html").unwrap();
1153        assert_eq!(
1154            extract_resolved_version(&u, "latest").as_deref(),
1155            Some("1.0.210")
1156        );
1157        let u2 = Url::parse("https://docs.rs/serde/latest/serde/index.html").unwrap();
1158        assert!(extract_resolved_version(&u2, "latest").is_none());
1159    }
1160
1161    #[test]
1162    fn module_url_template() {
1163        let segs = vec!["serde".into(), "de".into()];
1164        let u = get_item_url("serde", "latest", ItemKind::Module, &segs).unwrap();
1165        assert!(u.as_str().ends_with("/serde/de/index.html"));
1166    }
1167
1168    #[test]
1169    fn empty_html_readme() {
1170        let (md, empty) = extract_readme_markdown_from_html("<html><body></body></html>").unwrap();
1171        assert!(empty);
1172        assert!(md.is_empty());
1173    }
1174
1175    #[test]
1176    fn get_item_empty_segments_errors() {
1177        let err = get_item_url("clap", "latest", ItemKind::Struct, &[]).unwrap_err();
1178        assert_eq!(err.kind(), ErrorKind::InvalidInput);
1179    }
1180
1181    #[test]
1182    fn join_full_http_url_passthrough() {
1183        let base = Url::parse("https://docs.rs/serde/latest/serde/").unwrap();
1184        let full = "https://docs.rs/serde/latest/serde/struct.Error.html";
1185        assert_eq!(join_href(&base, full).unwrap(), full);
1186        let http = "http://example.invalid/x.html";
1187        assert_eq!(join_href(&base, http).unwrap(), http);
1188    }
1189
1190    #[test]
1191    fn origin_builders_trim_trailing_slash() {
1192        let u = readme_url_on_origin("https://docs.rs/", "demo", "1.0.0").unwrap();
1193        assert_eq!(u.as_str(), "https://docs.rs/demo/1.0.0/demo/index.html");
1194        let a = all_html_url_on_origin("https://docs.rs/", "demo", "1.0.0").unwrap();
1195        assert_eq!(a.as_str(), "https://docs.rs/demo/1.0.0/demo/all.html");
1196    }
1197
1198    #[test]
1199    fn empty_item_html_extract() {
1200        let (md, empty) = extract_item_markdown_from_html("<html><body></body></html>").unwrap();
1201        assert!(empty);
1202        assert!(md.is_empty());
1203    }
1204}