Skip to main content

docsrs_cli/docs_rs/
mod.rs

1//! docs.rs operations: readme, get-item, search-in-crate.
2//!
3//! # Extraction posture (Rules Rust — web scraping / data extraction)
4//!
5//! - **Transport** is HTTP GET ([`HttpClient`](crate::http::HttpClient)); this module only parses bytes.
6//! - **Structure** uses `scraper` (html5ever) + CSS selectors — never regex for
7//!   field extraction from HTML (version scrape uses path segments on `a[href]`).
8//! - **Sanitization** may use process-static regex only to strip `on*` handlers
9//!   and `javascript:` URLs after DOM script/style removal (XSS hygiene, not
10//!   content discovery).
11//! - **Markdown** conversion via `htmd` on the sanitized fragment.
12//! - **Provenance:** every success payload includes `source_url` (final URL).
13//!   Hit URLs from `all.html` join against that `source_url` base (stdlib /
14//!   mock / docs.rs) — never a hardcoded docs.rs host. Absolute hit hrefs must
15//!   stay **same-origin** as `source_url` (SCRAPE-S-001); off-origin are skipped.
16//! - **Encoding:** UTF-8 only after BOM strip ([`crate::http::decode_utf8`]);
17//!   docs.rs / doc.rust-lang.org ship UTF-8. `encoding_rs` multi-charset is OOS.
18//! - **Not a crawl:** no link frontier, no sitemap, no RSS/Atom.
19//! - **robots.txt:** **PROIBIDO** — this product never fetches, parses, or
20//!   enforces REP (operator mandate + ADR 0003). One-shot allowlisted GETs only.
21//!
22//! # Workload classification (Rules Rust — parallelism)
23//!
24//! - **I/O stage:** HTTPS GET via [`HttpClient`](crate::http::HttpClient) on multi-thread Tokio workers.
25//! - **CPU stage:** HTML scrape + markdown conversion via
26//!   [`ConcurrencyBudget::run_cpu_bound`](crate::concurrency::ConcurrencyBudget::run_cpu_bound)
27//!   (`spawn_blocking` + Semaphore).
28//! - **Hit scan:** `all.html` candidate lists are scanned sequentially at every
29//!   size. Per-item work is one URL resolve plus a string match — too small to
30//!   amortize thread fan-out and join. A `rayon` path was benchmarked from 16 to
31//!   32768 candidates, lost at every size (best case 0.85x), and was removed.
32
33pub(crate) mod assoc;
34mod fetch;
35pub(crate) mod hits;
36mod html;
37mod search;
38mod types;
39mod urls;
40
41pub use assoc::{
42    ASSOC_ANCHOR_MISS_PREFIX, AssocAnchorKind, METHOD_ANCHOR_PREFIXES, assoc_from_fragment,
43    associated_item_path, strip_assoc_anchor_prefix,
44};
45pub use fetch::{
46    fetch_item, fetch_item_at, fetch_item_at_with_echo, fetch_item_on_origin,
47    fetch_item_on_origin_with_echo, fetch_readme, fetch_readme_at, fetch_readme_on_origin,
48};
49pub use html::{
50    MethodExtract, extract_item_markdown_from_html, extract_method_markdown_from_html,
51    extract_method_markdown_scoped, extract_readme_markdown_from_html,
52    list_assoc_anchor_names_from_html, list_method_anchor_names,
53    list_method_anchor_names_from_html, sanitize_html_fragment, scrub_rustdoc_chrome,
54    strip_method_anchor_prefix,
55};
56pub use search::{
57    join_href, parse_all_html_hits, resolve_hit_url, search_in_crate, search_in_crate_at,
58    search_in_crate_from_html, search_in_crate_on_origin,
59};
60pub use types::{GetItemData, ReadmeData, SearchInCrateData, SearchInCrateHit};
61pub use urls::{
62    all_html_url, all_html_url_on_origin, get_item_url, get_item_url_on_origin,
63    get_item_url_on_origin_with_parent_kind, is_method_path, method_segments_from_parent,
64    pick_unique_type_path, readme_url, readme_url_on_origin, strip_crate_prefix_segments,
65};
66
67#[cfg(test)]
68mod tests;