docsrs_cli/docs_rs/mod.rs
1//! docs.rs operations: readme, get-item, search-in-crate.
2//!
3//! # Extraction posture (Rules Rust — web scraping / data extraction)
4//!
5//! - **Transport** is HTTP GET ([`HttpClient`](crate::http::HttpClient)); this module only parses bytes.
6//! - **Structure** uses `scraper` (html5ever) + CSS selectors — never regex for
7//! field extraction from HTML (version scrape uses path segments on `a[href]`).
8//! - **Sanitization** may use process-static regex only to strip `on*` handlers
9//! and `javascript:` URLs after DOM script/style removal (XSS hygiene, not
10//! content discovery).
11//! - **Markdown** conversion via `htmd` on the sanitized fragment.
12//! - **Provenance:** every success payload includes `source_url` (final URL).
13//! Hit URLs from `all.html` join against that `source_url` base (stdlib /
14//! mock / docs.rs) — never a hardcoded docs.rs host. Absolute hit hrefs must
15//! stay **same-origin** as `source_url` (SCRAPE-S-001); off-origin are skipped.
16//! - **Encoding:** UTF-8 only after BOM strip ([`crate::http::decode_utf8`]);
17//! docs.rs / doc.rust-lang.org ship UTF-8. `encoding_rs` multi-charset is OOS.
18//! - **Not a crawl:** no link frontier, no sitemap, no RSS/Atom.
19//! - **robots.txt:** **PROIBIDO** — this product never fetches, parses, or
20//! enforces REP (operator mandate + ADR 0003). One-shot allowlisted GETs only.
21//!
22//! # Workload classification (Rules Rust — parallelism)
23//!
24//! - **I/O stage:** HTTPS GET via [`HttpClient`](crate::http::HttpClient) on multi-thread Tokio workers.
25//! - **CPU stage:** HTML scrape + markdown conversion via
26//! [`ConcurrencyBudget::run_cpu_bound`](crate::concurrency::ConcurrencyBudget::run_cpu_bound)
27//! (`spawn_blocking` + Semaphore).
28//! - **Hit scan:** `all.html` candidate lists are scanned sequentially at every
29//! size. Per-item work is one URL resolve plus a string match — too small to
30//! amortize thread fan-out and join. A `rayon` path was benchmarked from 16 to
31//! 32768 candidates, lost at every size (best case 0.85x), and was removed.
32
33pub(crate) mod assoc;
34mod fetch;
35pub(crate) mod hits;
36mod html;
37mod search;
38mod types;
39mod urls;
40
41pub use assoc::{
42 ASSOC_ANCHOR_MISS_PREFIX, AssocAnchorKind, METHOD_ANCHOR_PREFIXES, assoc_from_fragment,
43 associated_item_path, strip_assoc_anchor_prefix,
44};
45pub use fetch::{
46 fetch_item, fetch_item_at, fetch_item_at_with_echo, fetch_item_on_origin,
47 fetch_item_on_origin_with_echo, fetch_readme, fetch_readme_at, fetch_readme_on_origin,
48};
49pub use html::{
50 MethodExtract, extract_item_markdown_from_html, extract_method_markdown_from_html,
51 extract_method_markdown_scoped, extract_readme_markdown_from_html,
52 list_assoc_anchor_names_from_html, list_method_anchor_names,
53 list_method_anchor_names_from_html, sanitize_html_fragment, scrub_rustdoc_chrome,
54 strip_method_anchor_prefix,
55};
56pub use search::{
57 join_href, parse_all_html_hits, resolve_hit_url, search_in_crate, search_in_crate_at,
58 search_in_crate_from_html, search_in_crate_on_origin,
59};
60pub use types::{GetItemData, ReadmeData, SearchInCrateData, SearchInCrateHit};
61pub use urls::{
62 all_html_url, all_html_url_on_origin, get_item_url, get_item_url_on_origin,
63 get_item_url_on_origin_with_parent_kind, is_method_path, method_segments_from_parent,
64 pick_unique_type_path, readme_url, readme_url_on_origin, strip_crate_prefix_segments,
65};
66
67#[cfg(test)]
68mod tests;