keyhog_sources/http.rs
1//! Shared HTTP client builder for every source / verifier site that
2//! goes over the network.
3//!
4//! Why this lives in one module
5//! ----------------------------
6//! `web.rs`, `github_org.rs`, the verifier's `verify/request.rs` and
7//! `verify/mod.rs`, and the slack source all build their own
8//! `reqwest::Client` directly. Without a shared builder, an operator
9//! who puts `HTTP_PROXY=http://burp:8080` into their environment would
10//! discover that *some* sites honor it (the ones that don't call
11//! `.no_proxy()`) and *others* don't, with no good signal as to why.
12//! Worse, adding `--proxy` to one site without the others would mean
13//! the verifier silently bypasses the proxy that the scan sources are
14//! routed through - so leaked-then-verified findings still leak the
15//! credential straight to the upstream API.
16//!
17//! [`HttpClientConfig`] is the single point of policy and
18//! [`blocking_client_builder`] / [`async_client_builder`] are the
19//! single construction sites. Every HTTP call in the binary flows
20//! through one of them.
21//!
22//! Policy summary
23//! --------------
24//! * **Proxy resolution.** ONLY explicit config (`HttpClientConfig::proxy`,
25//! set by the `--proxy` CLI flag / TOML) is honored. No environment
26//! variable sets or changes the proxy, and the builders call `.no_proxy()`
27//! when none is configured so reqwest's ambient `HTTPS_PROXY` /
28//! `HTTP_PROXY` / `ALL_PROXY` auto-detection can't silently reroute
29//! secret-bearing traffic. `--proxy off` disables proxying entirely.
30//! * **TLS verification.** On by default. ONLY `--insecure` (CLI / TOML)
31//! accepts an invalid certificate - needed for Burp / mitmproxy
32//! interception. No environment variable can disable verification: an
33//! ambient toggle must never turn off the protection guarding exfiltrated
34//! secrets from a MITM.
35//! * **Body-bomb defenses.** Auto-decompression OFF (the per-site
36//! chunk handlers opt in where they need it). 5-hop redirect cap.
37//! 30 s total per-request timeout default (no separate connect timeout;
38//! the total budget bounds the whole request including connect).
39//! * **User-Agent.** `keyhog/<version>` so operators can spot keyhog
40//! traffic in their proxy logs without grepping for "reqwest".
41
42use std::time::Duration;
43
44/// Single source of truth for outbound HTTP policy.
45#[derive(Debug, Clone, Default)]
46pub struct HttpClientConfig {
47 /// Explicit proxy URL. Overrides env-var detection. Accepts
48 /// `http://`, `https://`, `socks5://` schemes (forwarded to
49 /// reqwest unchanged). The literal string `"off"` disables
50 /// proxying entirely - including env-var inheritance.
51 pub proxy: Option<String>,
52 /// Accept invalid / self-signed TLS certs (Burp CA, mitmproxy CA).
53 /// Off by default.
54 pub insecure_tls: bool,
55 /// Optional per-request timeout override. `None` falls back to the ONE
56 /// shared default `crate::timeouts::HTTP_REQUEST` (30 s).
57 pub timeout: Option<Duration>,
58 /// User-Agent suffix appended after `keyhog/<version>`. Lets a
59 /// per-source caller add its own identifier (e.g. `web`,
60 /// `github-org`) without forcing every site to spell out the full
61 /// version string.
62 pub ua_suffix: Option<String>,
63 /// Allow cloud endpoints (`--s3-endpoint`, GCS / Azure container URLs) whose
64 /// literal host OR resolved address is private / loopback / link-local /
65 /// cloud-metadata. OFF by default: the SSRF host-screen in
66 /// `cloud::parse_http_endpoint` refuses every such endpoint. This is a
67 /// Tier-A config knob (set by `--allow-private-cloud-endpoint` / TOML, NEVER
68 /// an environment variable, scan/security behavior is resolved config, not
69 /// env) for legit private-network deployments (MinIO / Ceph on an internal
70 /// gateway) and loopback mock servers in integration tests.
71 pub allow_private_endpoint: bool,
72}
73
74impl HttpClientConfig {
75 /// The configured proxy, or `None`. ONLY the explicit `proxy` field (set by
76 /// the `--proxy` CLI flag / TOML) is honored, no environment variable can
77 /// change egress routing (config-policy mandate: env never overrides
78 /// behavior). Ambient `HTTP(S)_PROXY` is separately neutralized in the
79 /// builders via `.no_proxy()` so a stray CI/shell proxy can't silently
80 /// reroute the secret-verification traffic. `Some("off")` disables proxying.
81 #[cfg(any(
82 feature = "azure",
83 feature = "web",
84 feature = "github",
85 feature = "gitlab",
86 feature = "bitbucket",
87 feature = "slack",
88 feature = "s3",
89 feature = "gcs"
90 ))]
91 pub(crate) fn effective_proxy(&self) -> Option<String> {
92 self.proxy.clone()
93 }
94
95 /// Effective timeout used by both shared HTTP clients and pre-connect
96 /// network policy checks such as WebSource DNS screening.
97 #[cfg(any(
98 feature = "azure",
99 feature = "web",
100 feature = "github",
101 feature = "gitlab",
102 feature = "bitbucket",
103 feature = "slack",
104 feature = "s3",
105 feature = "gcs"
106 ))]
107 pub(crate) fn effective_timeout(&self) -> Duration {
108 self.timeout.unwrap_or(crate::timeouts::HTTP_REQUEST) // LAW10: Tier-A config default, unset timeout uses the ONE shared HTTP_REQUEST default, not a swallowed error
109 }
110
111 /// Whether to accept invalid / self-signed TLS certs. ONLY the explicit
112 /// `insecure_tls` field (set by `--insecure` / TOML) is honored, no
113 /// environment variable can disable certificate verification. An ambient
114 /// toggle must never be able to switch off the only thing protecting
115 /// exfiltrated secrets from a MITM on the verifier's outbound calls.
116 #[cfg(any(
117 feature = "azure",
118 feature = "web",
119 feature = "github",
120 feature = "gitlab",
121 feature = "bitbucket",
122 feature = "slack",
123 feature = "s3",
124 feature = "gcs"
125 ))]
126 pub(crate) fn effective_insecure_tls(&self) -> bool {
127 self.insecure_tls
128 }
129
130 /// HTTP config identical to the default EXCEPT it permits private / loopback
131 /// endpoints. The config-flag replacement for the retired
132 /// `KEYHOG_ALLOW_PRIVATE_CLOUD_ENDPOINT` env opt-in, used by the source-test
133 /// facade so httpmock (`127.0.0.1`) endpoints pass the cloud SSRF screen
134 /// without any process-global env state.
135 #[cfg(any(feature = "s3", feature = "gcs", feature = "azure"))]
136 pub(crate) fn allowing_private_endpoint() -> Self {
137 Self {
138 allow_private_endpoint: true,
139 ..Self::default()
140 }
141 }
142}
143
144#[cfg(any(
145 feature = "azure",
146 feature = "web",
147 feature = "github",
148 feature = "gitlab",
149 feature = "bitbucket",
150 feature = "slack",
151 feature = "s3",
152 feature = "gcs"
153))]
154pub(crate) const REDIRECT_LIMIT: usize = 5;
155
156/// Extract the bare media type from a `Content-Type` header value: the text
157/// before the first `;` (dropping any `charset=`/`boundary=` parameters), with
158/// surrounding whitespace trimmed. Single owner in this always-compiled module
159/// so every content-type classifier, the `web` response router
160/// (`crate::web`) AND the cloud binary/unknown checks (`crate::cloud`), splits
161/// the header the same way WITHOUT the `web` feature having to pull in a cloud
162/// provider feature (the fn used to live in the `cfg`-gated `cloud` module,
163/// which made `--features web` fail to compile standalone). Gated to exactly
164/// the features whose code calls it (`web`, and the cloud providers behind
165/// `crate::cloud`) so a minimal build carries no dead helper.
166#[cfg(any(feature = "web", feature = "azure", feature = "s3", feature = "gcs"))]
167pub(crate) fn media_type(content_type: &str) -> &str {
168 content_type
169 .split_once(';')
170 .map_or(content_type, |(media_type, _)| media_type)
171 .trim()
172}
173
174pub(crate) fn user_agent(suffix: Option<&str>) -> String {
175 let base = concat!("keyhog/", env!("CARGO_PKG_VERSION"));
176 match suffix {
177 Some(s) if !s.is_empty() => format!("{base} ({s})"),
178 _ => base.to_string(),
179 }
180}
181
182/// The ONE outbound-HTTP policy, applied identically to the blocking and async
183/// `ClientBuilder`s (which share every method name but no common trait, so a
184/// macro is the single owner). Any new hardening, a decompression toggle, a
185/// proxy sentinel, a TLS knob, is added here once and both builders inherit it,
186/// preventing the two-copies drift this module exists to prevent.
187///
188/// `$builder` is a fresh `reqwest::{blocking,}::ClientBuilder`; expands inside a
189/// `-> Result<_, String>` fn so the proxy-parse `?` propagates.
190#[cfg(any(
191 feature = "azure",
192 feature = "web",
193 feature = "github",
194 feature = "gitlab",
195 feature = "bitbucket",
196 feature = "slack",
197 feature = "s3",
198 feature = "gcs"
199))]
200macro_rules! shared_http_policy {
201 ($builder:expr, $cfg:expr) => {{
202 let cfg = $cfg;
203 let builder = $builder
204 .timeout(cfg.effective_timeout()) // LAW10: unset timeout uses the ONE shared HTTP_REQUEST default, not a silent error
205 .redirect(reqwest::redirect::Policy::limited(REDIRECT_LIMIT))
206 .user_agent(user_agent(cfg.ua_suffix.as_deref()))
207 .no_gzip()
208 .no_brotli()
209 .no_deflate()
210 .danger_accept_invalid_certs(cfg.effective_insecure_tls());
211
212 // Ambient HTTPS_PROXY / HTTP_PROXY / ALL_PROXY is always neutralized via
213 // `.no_proxy()`; ONLY an explicit `--proxy` / TOML URL routes traffic, so
214 // a stray CI/shell proxy can't silently reroute secret-bearing requests.
215 let builder = match cfg.effective_proxy().as_deref() {
216 Some("off") | Some("none") | Some("") | None => builder.no_proxy(),
217 Some(url) => {
218 let proxy = reqwest::Proxy::all(url)
219 .map_err(|e| format!("invalid --proxy URL {url:?}: {e}"))?;
220 builder.proxy(proxy)
221 }
222 };
223 Ok(builder)
224 }};
225}
226
227/// Build a `reqwest::blocking::ClientBuilder` populated with the
228/// shared policy. Callers can chain extra builder methods (e.g.
229/// `.default_headers(...)`) before `.build()`.
230#[cfg(any(
231 feature = "azure",
232 feature = "web",
233 feature = "github",
234 feature = "gitlab",
235 feature = "bitbucket",
236 feature = "slack",
237 feature = "s3",
238 feature = "gcs"
239))]
240pub(crate) fn blocking_client_builder(
241 cfg: &HttpClientConfig,
242) -> Result<reqwest::blocking::ClientBuilder, String> {
243 shared_http_policy!(reqwest::blocking::Client::builder(), cfg)
244}
245
246/// Async sibling for the verifier's tokio-based call sites.
247#[cfg(any(
248 feature = "azure",
249 feature = "web",
250 feature = "github",
251 feature = "gitlab",
252 feature = "bitbucket",
253 feature = "slack",
254 feature = "s3",
255 feature = "gcs"
256))]
257pub(crate) fn async_client_builder(
258 cfg: &HttpClientConfig,
259) -> Result<reqwest::ClientBuilder, String> {
260 shared_http_policy!(reqwest::Client::builder(), cfg)
261}