Skip to main content

docsrs_cli/http/
mod.rs

1//! HTTP client: rustls, retry, rate limit, body cap, host allowlist, cancel-aware.
2//!
3//! # Product fetch posture (Rules Rust — rede + web scraping)
4//!
5//! This is a **one-shot docs client**, not a general crawler or HTTP server:
6//! - **GET-only** product surface against a fixed HTTPS host allowlist (SSRF gate).
7//! - **One primary request** per command (no multi-URL frontier, no sitemap/RSS).
8//! - **User-Agent** identifies `docsrs-cli/{version}` plus optional contact.
9//! - **Politeness:** per-host delay floor + additive jitter ([`crate::retry::politeness_delay`]),
10//!   in-process clock and cross-process lock+stamp.
11//! - **Body budget:** stream with `Content-Length` early reject + `try_reserve*`
12//!   (no unbounded `bytes().await`).
13//! - **TLS:** rustls only; provider, trust anchors and version floor are the
14//!   constants `content_type::TLS_CRYPTO_PROVIDER`,
15//!   `content_type::TLS_TRUST_ANCHORS` and
16//!   `content_type::RUSTLS_VERSION_FLOOR`, never repeated as literals here;
17//!   min TLS 1.2; never `danger_accept_invalid_certs` (ADR 0007).
18//! - **HTTP/2** enabled (feature `http2`); ALPN negotiates with the origin.
19//! - **Proxy:** system env `HTTP_PROXY` / `HTTPS_PROXY` / `ALL_PROXY` / `NO_PROXY`
20//!   via reqwest `system-proxy` (allowlist still applies to the **target** URL).
21//! - **Sockets:** `TCP_NODELAY` on; keepalive idle/interval/retries named constants.
22//! - **Pool:** tiny idle pool sized for one-shot multi-host GETs (not a daemon).
23//! - **No cookie jar** (`cookie_store` feature not enabled); no shared cookies.
24//! - **Provenance:** HTML payloads expose `source_url` on success envelopes.
25//! - **robots.txt:** **PROIBIDO** — never fetch/parse/enforce REP (ADR 0003).
26//! - **OOS:** ETag conditional GETs, anti-bot fingerprinting, headless Chrome,
27//!   proxy rotation, DoH/DoT custom resolver, server bind/accept, circuit
28//!   breaker, mTLS. See `docs/decisions/0003-web-fetch-scope.md`.
29//!
30//! # Workload classification (Rules Rust — parallelism + memória)
31//!
32//! - **Class:** mixed — async I/O stage on multi-thread Tokio + CPU parse stage
33//!   via [`crate::concurrency::ConcurrencyBudget::run_cpu_bound`].
34//! - **Shareable:** methods take `&self`. Per-host in-process clock is a
35//!   `Mutex<HashMap>` with sleep **outside** the lock (no `Mutex` across `.await`
36//!   for the map itself; cross-process flock still owns multi-process throttle).
37//! - **Budget:** each client owns a [`ConcurrencyBudget`](crate::concurrency::ConcurrencyBudget) for `spawn_blocking`
38//!   parse work (and future multi-GET fan-out). Bound is explicit (auto or
39//!   `--max-concurrency`).
40//! - **Network:** product commands still issue one primary GET at a time; rate
41//!   limit + body stream cap remain the I/O backpressure.
42//!
43//! # Retry (Rules Rust — retry/backoff)
44//!
45//! Policy lives in [`crate::retry::RetryConfig`] (not inline magic). GET-only
46//! product surface ⇒ idempotent retries for 408/429/5xx/transport only. Dual
47//! budget: `max_attempts` + `max_elapsed_ms`. `Retry-After` accepts delta-seconds
48//! and HTTP-date. Kill switch `--disable-retry` / TOML `disable_retry` /
49//! `max_retries=0`. See module docs on `retry`.
50//!
51//! Layout (SRP): `client` · `body` · `content_type` · `allowlist` ·
52//! `rate_limit` · `constants` (all crate-internal).
53
54mod allowlist;
55mod body;
56mod client;
57mod constants;
58mod content_type;
59mod rate_limit;
60mod tls;
61
62pub use rate_limit::RATE_LIMIT_DIR_NAME;
63
64pub(crate) use allowlist::is_allowed_host;
65pub use body::decode_utf8;
66pub use client::{HttpClient, HttpResponse};
67pub use constants::{
68    POOL_IDLE_TIMEOUT_SECS, POOL_MAX_IDLE_PER_HOST, TCP_KEEPALIVE_IDLE_SECS,
69    TCP_KEEPALIVE_INTERVAL_SECS, TCP_KEEPALIVE_RETRIES,
70};
71pub use content_type::{
72    client_posture_detail, content_type_looks_html, content_type_looks_json,
73    require_content_type_html, require_content_type_json,
74};
75
76#[cfg(test)]
77mod tests {
78    use super::content_type::ascii_contains_ignore_case;
79    use super::*;
80    use crate::config::Config;
81    use crate::error::ErrorKind;
82    use crate::shutdown::CancelFlag;
83    use bytes::Bytes;
84    use reqwest::Url;
85    use std::sync::Once;
86
87    /// reqwest 0.13 `rustls-no-provider` needs a process CryptoProvider (ring; ADR 0007).
88    /// Binary installs in `main`; lib unit tests install once here.
89    fn ensure_test_provider() {
90        static ONCE: Once = Once::new();
91        ONCE.call_once(|| {
92            let _ = rustls::crypto::ring::default_provider().install_default();
93        });
94    }
95
96    #[test]
97    fn allowlist_https_only() {
98        let ok = Url::parse("https://docs.rs/serde").unwrap();
99        assert!(is_allowed_host(&ok, false));
100        let stdlib = Url::parse("https://doc.rust-lang.org/stable/std/").unwrap();
101        assert!(is_allowed_host(&stdlib, false));
102        let bad = Url::parse("http://docs.rs/serde").unwrap();
103        assert!(!is_allowed_host(&bad, false));
104        let evil = Url::parse("https://evil.example/").unwrap();
105        assert!(!is_allowed_host(&evil, false));
106    }
107
108    #[test]
109    fn utf8_bom_strip() {
110        let mut v = vec![0xEF, 0xBB, 0xBF];
111        v.extend_from_slice(b"hello");
112        let s = decode_utf8(&Bytes::from(v)).unwrap();
113        assert_eq!(s, "hello");
114    }
115
116    #[test]
117    fn content_types() {
118        assert!(content_type_looks_json(Some(
119            "application/json; charset=utf-8"
120        )));
121        assert!(content_type_looks_json(Some("Application/JSON")));
122        assert!(content_type_looks_html(Some("text/html")));
123        assert!(content_type_looks_html(Some("Text/HTML; charset=utf-8")));
124        assert!(content_type_looks_html(Some("application/xhtml+xml")));
125        assert!(!content_type_looks_json(Some("text/html")));
126        assert!(!content_type_looks_json(None));
127        assert!(!content_type_looks_html(None));
128        assert!(require_content_type_json(None).is_ok());
129        assert!(require_content_type_json(Some("application/json")).is_ok());
130        assert!(require_content_type_json(Some("text/html")).is_err());
131        assert!(require_content_type_html(None).is_ok());
132        assert!(require_content_type_html(Some("text/html")).is_ok());
133        assert!(require_content_type_html(Some("application/json")).is_err());
134    }
135
136    #[test]
137    fn ascii_contains_ignore_case_edges() {
138        assert!(ascii_contains_ignore_case("abJSON", "json"));
139        assert!(ascii_contains_ignore_case("json", "json"));
140        assert!(!ascii_contains_ignore_case("js", "json"));
141        assert!(ascii_contains_ignore_case("x", ""));
142    }
143
144    #[test]
145    fn retry_policy_kill_switch() {
146        ensure_test_provider();
147        let cfg = Config {
148            disable_retry: true,
149            max_retries: 5,
150            ..Config::default()
151        };
152        let http = HttpClient::new(cfg, CancelFlag::new()).unwrap();
153        assert!(!http.retry_policy().enabled);
154        assert_eq!(http.retry_policy().max_attempts(), 1);
155    }
156
157    #[test]
158    fn invalid_user_agent_rejected() {
159        let cfg = Config {
160            user_agent: "bad\nua".into(),
161            ..Config::default()
162        };
163        match HttpClient::new(cfg, CancelFlag::new()) {
164            Ok(_) => panic!("expected invalid user-agent to fail"),
165            Err(err) => assert_eq!(err.kind(), ErrorKind::Config),
166        }
167    }
168
169    #[test]
170    fn localhost_requires_allow_loopback_flag() {
171        let ok = Url::parse("http://127.0.0.1:9/x").unwrap();
172        assert!(!is_allowed_host(&ok, false));
173        assert!(is_allowed_host(&ok, true));
174        let local = Url::parse("http://localhost:9/x").unwrap();
175        assert!(!is_allowed_host(&local, false));
176        assert!(is_allowed_host(&local, true));
177    }
178
179    #[test]
180    fn decode_utf8_rejects_invalid() {
181        let err = decode_utf8(&Bytes::from(vec![0xff, 0xfe])).unwrap_err();
182        assert_eq!(err.kind(), ErrorKind::Parse);
183    }
184
185    #[test]
186    fn named_socket_and_pool_constants_are_sane() {
187        // Exact product posture (one-shot CLI) — change only with ADR.
188        assert_eq!(POOL_MAX_IDLE_PER_HOST, 2);
189        assert_eq!(POOL_IDLE_TIMEOUT_SECS, 30);
190        assert_eq!(TCP_KEEPALIVE_IDLE_SECS, 60);
191        assert_eq!(TCP_KEEPALIVE_INTERVAL_SECS, 10);
192        assert_eq!(TCP_KEEPALIVE_RETRIES, 3);
193    }
194
195    #[test]
196    fn client_posture_mentions_tls_http2_and_proxy() {
197        let d = client_posture_detail();
198        assert!(d.contains("rustls"));
199        assert!(d.contains("provider=ring"));
200        // `doctor` must disclose that this binary carries a C build dependency,
201        // and why the pure-Rust replacements were rejected. A Rust-native claim
202        // that is only true in the source is not auditable from the binary.
203        assert!(d.contains("c-build-dep(ring)"));
204        assert!(d.contains("pure-rust-blocked"));
205        // Read from the constant, not repeated as a literal: asserting the
206        // literal here verified the string against a copy of itself and looked
207        // like coverage while proving nothing about the manifest pin.
208        assert!(d.contains(content_type::RUSTLS_VERSION_FLOOR));
209        assert!(d.contains("http2"));
210        assert!(d.contains("system-proxy"));
211        assert!(d.contains("tcp_nodelay"));
212        assert!(d.contains("host-allowlist"));
213    }
214
215    #[test]
216    fn client_builds_with_network_posture() {
217        ensure_test_provider();
218        let cfg = Config {
219            rate_limit_delay_ms: 0,
220            ..Config::default()
221        };
222        let http = HttpClient::new(cfg, CancelFlag::new()).expect("client builds");
223        assert!(http.retry_policy().enabled);
224        assert!(http.budget().max() >= 1);
225    }
226}