Skip to main content

docsrs_cli/http/
mod.rs

1//! HTTP client: rustls, retry, rate limit, body cap, host allowlist, cancel-aware.
2//!
3//! # Product fetch posture (Rules Rust — rede + web scraping)
4//!
5//! This is a **one-shot docs client**, not a general crawler or HTTP server:
6//! - **GET-only** product surface against a fixed HTTPS host allowlist (SSRF gate).
7//! - **One primary request** per command (no multi-URL frontier, no sitemap/RSS).
8//! - **User-Agent** identifies `docsrs-cli/{version}` plus optional contact.
9//! - **Politeness:** per-host delay floor + additive jitter ([`crate::retry::politeness_delay`]),
10//!   in-process clock and cross-process lock+stamp.
11//! - **Body budget:** stream with `Content-Length` early reject + `try_reserve*`
12//!   (no unbounded `bytes().await`).
13//! - **TLS:** rustls only (`provider=ring`, floor ≥0.23.18); min TLS 1.2;
14//!   webpki-roots; never `danger_accept_invalid_certs` (ADR 0007).
15//! - **HTTP/2** enabled (feature `http2`); ALPN negotiates with the origin.
16//! - **Proxy:** system env `HTTP_PROXY` / `HTTPS_PROXY` / `ALL_PROXY` / `NO_PROXY`
17//!   via reqwest `system-proxy` (allowlist still applies to the **target** URL).
18//! - **Sockets:** `TCP_NODELAY` on; keepalive idle/interval/retries named constants.
19//! - **Pool:** tiny idle pool sized for one-shot multi-host GETs (not a daemon).
20//! - **No cookie jar** (`cookie_store` feature not enabled); no shared cookies.
21//! - **Provenance:** HTML payloads expose `source_url` on success envelopes.
22//! - **robots.txt:** **PROIBIDO** — never fetch/parse/enforce REP (ADR 0003).
23//! - **OOS:** ETag conditional GETs, anti-bot fingerprinting, headless Chrome,
24//!   proxy rotation, DoH/DoT custom resolver, server bind/accept, circuit
25//!   breaker, mTLS. See `docs/decisions/0003-web-fetch-scope.md`.
26//!
27//! # Workload classification (Rules Rust — parallelism + memória)
28//!
29//! - **Class:** mixed — async I/O stage on multi-thread Tokio + CPU parse stage
30//!   via [`crate::concurrency::ConcurrencyBudget::run_cpu_bound`].
31//! - **Shareable:** methods take `&self`. Per-host in-process clock is a
32//!   `Mutex<HashMap>` with sleep **outside** the lock (no `Mutex` across `.await`
33//!   for the map itself; cross-process flock still owns multi-process throttle).
34//! - **Budget:** each client owns a [`ConcurrencyBudget`] for `spawn_blocking`
35//!   parse work (and future multi-GET fan-out). Bound is explicit (auto or
36//!   `--max-concurrency`).
37//! - **Network:** product commands still issue one primary GET at a time; rate
38//!   limit + body stream cap remain the I/O backpressure.
39//!
40//! # Retry (Rules Rust — retry/backoff)
41//!
42//! Policy lives in [`crate::retry::RetryConfig`] (not inline magic). GET-only
43//! product surface ⇒ idempotent retries for 408/429/5xx/transport only. Dual
44//! budget: `max_attempts` + `max_elapsed_ms`. `Retry-After` accepts delta-seconds
45//! and HTTP-date. Kill switch `--disable-retry` / TOML `disable_retry` /
46//! `max_retries=0`. See module docs on `retry`.
47//!
48//! Layout (SRP): [`client`] · [`body`] · [`content_type`] · [`allowlist`] ·
49//! [`rate_limit`] · [`constants`].
50
51mod allowlist;
52mod body;
53mod client;
54mod constants;
55mod content_type;
56mod rate_limit;
57
58pub(crate) use allowlist::is_allowed_host;
59pub use body::decode_utf8;
60pub use client::{HttpClient, HttpResponse};
61pub use constants::{
62    POOL_IDLE_TIMEOUT_SECS, POOL_MAX_IDLE_PER_HOST, TCP_KEEPALIVE_IDLE_SECS,
63    TCP_KEEPALIVE_INTERVAL_SECS, TCP_KEEPALIVE_RETRIES,
64};
65pub use content_type::{
66    client_posture_detail, content_type_looks_html, content_type_looks_json,
67    require_content_type_html, require_content_type_json,
68};
69
70#[cfg(test)]
71mod tests {
72    use super::content_type::ascii_contains_ignore_case;
73    use super::*;
74    use crate::config::Config;
75    use crate::error::ErrorKind;
76    use crate::shutdown::CancelFlag;
77    use bytes::Bytes;
78    use reqwest::Url;
79    use std::sync::Once;
80
81    /// reqwest 0.13 `rustls-no-provider` needs a process CryptoProvider (ring; ADR 0007).
82    /// Binary installs in `main`; lib unit tests install once here.
83    fn ensure_ring_provider() {
84        static ONCE: Once = Once::new();
85        ONCE.call_once(|| {
86            let _ = rustls::crypto::ring::default_provider().install_default();
87        });
88    }
89
90    #[test]
91    fn allowlist_https_only() {
92        let ok = Url::parse("https://docs.rs/serde").unwrap();
93        assert!(is_allowed_host(&ok, false));
94        let stdlib = Url::parse("https://doc.rust-lang.org/stable/std/").unwrap();
95        assert!(is_allowed_host(&stdlib, false));
96        let bad = Url::parse("http://docs.rs/serde").unwrap();
97        assert!(!is_allowed_host(&bad, false));
98        let evil = Url::parse("https://evil.example/").unwrap();
99        assert!(!is_allowed_host(&evil, false));
100    }
101
102    #[test]
103    fn utf8_bom_strip() {
104        let mut v = vec![0xEF, 0xBB, 0xBF];
105        v.extend_from_slice(b"hello");
106        let s = decode_utf8(&Bytes::from(v)).unwrap();
107        assert_eq!(s, "hello");
108    }
109
110    #[test]
111    fn content_types() {
112        assert!(content_type_looks_json(Some(
113            "application/json; charset=utf-8"
114        )));
115        assert!(content_type_looks_json(Some("Application/JSON")));
116        assert!(content_type_looks_html(Some("text/html")));
117        assert!(content_type_looks_html(Some("Text/HTML; charset=utf-8")));
118        assert!(content_type_looks_html(Some("application/xhtml+xml")));
119        assert!(!content_type_looks_json(Some("text/html")));
120        assert!(!content_type_looks_json(None));
121        assert!(!content_type_looks_html(None));
122        assert!(require_content_type_json(None).is_ok());
123        assert!(require_content_type_json(Some("application/json")).is_ok());
124        assert!(require_content_type_json(Some("text/html")).is_err());
125        assert!(require_content_type_html(None).is_ok());
126        assert!(require_content_type_html(Some("text/html")).is_ok());
127        assert!(require_content_type_html(Some("application/json")).is_err());
128    }
129
130    #[test]
131    fn ascii_contains_ignore_case_edges() {
132        assert!(ascii_contains_ignore_case("abJSON", "json"));
133        assert!(ascii_contains_ignore_case("json", "json"));
134        assert!(!ascii_contains_ignore_case("js", "json"));
135        assert!(ascii_contains_ignore_case("x", ""));
136    }
137
138    #[test]
139    fn retry_policy_kill_switch() {
140        ensure_ring_provider();
141        let cfg = Config {
142            disable_retry: true,
143            max_retries: 5,
144            ..Config::default()
145        };
146        let http = HttpClient::new(cfg, CancelFlag::new()).unwrap();
147        assert!(!http.retry_policy().enabled);
148        assert_eq!(http.retry_policy().max_attempts(), 1);
149    }
150
151    #[test]
152    fn invalid_user_agent_rejected() {
153        let cfg = Config {
154            user_agent: "bad\nua".into(),
155            ..Config::default()
156        };
157        match HttpClient::new(cfg, CancelFlag::new()) {
158            Ok(_) => panic!("expected invalid user-agent to fail"),
159            Err(err) => assert_eq!(err.kind(), ErrorKind::Config),
160        }
161    }
162
163    #[test]
164    fn localhost_requires_allow_loopback_flag() {
165        let ok = Url::parse("http://127.0.0.1:9/x").unwrap();
166        assert!(!is_allowed_host(&ok, false));
167        assert!(is_allowed_host(&ok, true));
168        let local = Url::parse("http://localhost:9/x").unwrap();
169        assert!(!is_allowed_host(&local, false));
170        assert!(is_allowed_host(&local, true));
171    }
172
173    #[test]
174    fn decode_utf8_rejects_invalid() {
175        let err = decode_utf8(&Bytes::from(vec![0xff, 0xfe])).unwrap_err();
176        assert_eq!(err.kind(), ErrorKind::Parse);
177    }
178
179    #[test]
180    fn named_socket_and_pool_constants_are_sane() {
181        // Exact product posture (one-shot CLI) — change only with ADR.
182        assert_eq!(POOL_MAX_IDLE_PER_HOST, 2);
183        assert_eq!(POOL_IDLE_TIMEOUT_SECS, 30);
184        assert_eq!(TCP_KEEPALIVE_IDLE_SECS, 60);
185        assert_eq!(TCP_KEEPALIVE_INTERVAL_SECS, 10);
186        assert_eq!(TCP_KEEPALIVE_RETRIES, 3);
187    }
188
189    #[test]
190    fn client_posture_mentions_tls_http2_and_proxy() {
191        let d = client_posture_detail();
192        assert!(d.contains("rustls"));
193        assert!(d.contains("provider=ring"));
194        assert!(d.contains("0.23.18"));
195        assert!(d.contains("http2"));
196        assert!(d.contains("system-proxy"));
197        assert!(d.contains("tcp_nodelay"));
198        assert!(d.contains("host-allowlist"));
199    }
200
201    #[test]
202    fn client_builds_with_network_posture() {
203        ensure_ring_provider();
204        let cfg = Config {
205            rate_limit_delay_ms: 0,
206            ..Config::default()
207        };
208        let http = HttpClient::new(cfg, CancelFlag::new()).expect("client builds");
209        assert!(http.retry_policy().enabled);
210        assert!(http.budget().max() >= 1);
211    }
212}