yldfi_common/
http.rs

1//! HTTP client utilities with proxy support
2//!
3//! This module provides shared HTTP client configuration and building
4//! functionality that can be used by all API crates in yldfi-rs.
5//!
6//! # Example
7//!
8//! ```no_run
9//! use yldfi_common::http::{HttpClientConfig, build_client};
10//!
11//! let config = HttpClientConfig::default()
12//!     .with_proxy("http://user:pass@proxy:8080")
13//!     .with_timeout_secs(60);
14//!
15//! let client = build_client(&config).unwrap();
16//! ```
17
18use reqwest::Client;
19use std::time::Duration;
20use thiserror::Error;
21use url::Url;
22
23/// Default User-Agent to avoid Cloudflare blocks.
24///
25/// Many API providers (especially those behind Cloudflare) block requests with
26/// default library User-Agent strings. This mimics a real browser to avoid
27/// 403 Forbidden responses.
28pub const DEFAULT_USER_AGENT: &str = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36";
29
30/// Default request timeout (30 seconds).
31///
32/// This balances responsiveness with reliability:
33/// - Short enough to fail fast on unresponsive servers
34/// - Long enough for slow API responses (e.g., archive node queries, large responses)
35/// - Matches common API gateway timeouts (AWS API Gateway: 29s, Cloudflare: 30s)
36pub const DEFAULT_TIMEOUT: Duration = Duration::from_secs(30);
37
38/// Redact credentials from a proxy URL for safe logging (MED-001 fix)
39///
40/// Replaces username and password with `[REDACTED]` if present.
41fn redact_proxy_url(url: &str) -> String {
42    if let Ok(mut parsed) = Url::parse(url) {
43        if !parsed.username().is_empty() || parsed.password().is_some() {
44            let _ = parsed.set_username("[REDACTED]");
45            let _ = parsed.set_password(Some("[REDACTED]"));
46        }
47        parsed.to_string()
48    } else {
49        "[invalid proxy URL]".to_string()
50    }
51}
52
53/// HTTP client configuration errors
54#[derive(Debug, Error)]
55pub enum HttpError {
56    #[error("Invalid proxy URL: {0}")]
57    InvalidProxy(String),
58
59    #[error("Failed to build HTTP client: {0}")]
60    BuildError(String),
61}
62
63impl From<reqwest::Error> for HttpError {
64    fn from(e: reqwest::Error) -> Self {
65        HttpError::BuildError(e.to_string())
66    }
67}
68
69/// Default connection pool idle timeout (90 seconds).
70///
71/// Connections are kept alive for reuse to avoid TCP/TLS handshake overhead.
72/// 90 seconds is chosen because:
73/// - Most HTTP keep-alive defaults are 60-120 seconds
74/// - Long enough to benefit from connection reuse in burst operations
75/// - Short enough to release resources reasonably quickly after idle periods
76/// - Matches common load balancer idle timeouts (AWS ALB: 60s, nginx: 75s)
77pub const DEFAULT_POOL_IDLE_TIMEOUT: Duration = Duration::from_secs(90);
78
79/// Default maximum idle connections per host (10).
80///
81/// This limits memory usage while allowing good parallelism:
82/// - 10 connections supports typical CLI concurrency (1-10 parallel requests)
83/// - Higher values waste memory for infrequent hosts
84/// - Lower values cause connection churn under parallel load
85/// - reqwest's default is 100, which is excessive for CLI tools
86///
87/// ## Tuning Guidelines (MED-004)
88///
89/// **Increase `DEFAULT_POOL_MAX_IDLE_PER_HOST` if:**
90/// - Running many parallel CLI commands to the same API
91/// - Seeing connection setup latency in traces
92/// - Batch operations with high parallelism (e.g., 20+ concurrent requests)
93///
94/// **Decrease if:**
95/// - Running on memory-constrained systems
96/// - Targeting many different API hosts (memory per host adds up)
97/// - Simple sequential operations with low parallelism
98///
99/// **Adjust `DEFAULT_POOL_IDLE_TIMEOUT` based on target API's keep-alive:**
100/// - Lower (30-60s) for APIs with short keep-alive timeouts
101/// - Higher (120s+) for APIs with long timeouts or when running long batch operations
102/// - Match to the minimum keep-alive of your target APIs to avoid broken connection errors
103pub const DEFAULT_POOL_MAX_IDLE_PER_HOST: usize = 10;
104
105/// HTTP client configuration
106#[derive(Debug, Clone)]
107pub struct HttpClientConfig {
108    /// Request timeout
109    pub timeout: Duration,
110    /// User-Agent header
111    pub user_agent: String,
112    /// Optional proxy URL
113    ///
114    /// Supports HTTP/HTTPS proxies with optional authentication:
115    /// - `http://proxy.example.com:8080`
116    /// - `http://user:password@proxy.example.com:8080`
117    ///
118    /// **Security Note**: Embedded credentials in the URL are supported.
119    /// Avoid logging proxy URLs directly as they may contain secrets.
120    pub proxy: Option<String>,
121    /// Connection pool idle timeout
122    pub pool_idle_timeout: Duration,
123    /// Maximum idle connections per host
124    pub pool_max_idle_per_host: usize,
125}
126
127impl Default for HttpClientConfig {
128    fn default() -> Self {
129        Self {
130            timeout: DEFAULT_TIMEOUT,
131            user_agent: DEFAULT_USER_AGENT.to_string(),
132            proxy: None,
133            pool_idle_timeout: DEFAULT_POOL_IDLE_TIMEOUT,
134            pool_max_idle_per_host: DEFAULT_POOL_MAX_IDLE_PER_HOST,
135        }
136    }
137}
138
139impl HttpClientConfig {
140    /// Create a new config with default settings
141    #[must_use]
142    pub fn new() -> Self {
143        Self::default()
144    }
145
146    /// Set request timeout
147    #[must_use]
148    pub fn with_timeout(mut self, timeout: Duration) -> Self {
149        self.timeout = timeout;
150        self
151    }
152
153    /// Set request timeout in seconds
154    #[must_use]
155    pub fn with_timeout_secs(mut self, secs: u64) -> Self {
156        self.timeout = Duration::from_secs(secs);
157        self
158    }
159
160    /// Set User-Agent header
161    #[must_use]
162    pub fn with_user_agent(mut self, user_agent: impl Into<String>) -> Self {
163        self.user_agent = user_agent.into();
164        self
165    }
166
167    /// Set proxy URL
168    #[must_use]
169    pub fn with_proxy(mut self, proxy: impl Into<String>) -> Self {
170        self.proxy = Some(proxy.into());
171        self
172    }
173
174    /// Set optional proxy URL
175    #[must_use]
176    pub fn with_optional_proxy(mut self, proxy: Option<String>) -> Self {
177        self.proxy = proxy;
178        self
179    }
180
181    /// Set connection pool idle timeout
182    #[must_use]
183    pub fn with_pool_idle_timeout(mut self, timeout: Duration) -> Self {
184        self.pool_idle_timeout = timeout;
185        self
186    }
187
188    /// Set maximum idle connections per host
189    #[must_use]
190    pub fn with_pool_max_idle_per_host(mut self, max: usize) -> Self {
191        self.pool_max_idle_per_host = max;
192        self
193    }
194}
195
196/// Build a reqwest Client with the given configuration
197pub fn build_client(config: &HttpClientConfig) -> Result<Client, HttpError> {
198    let mut builder = Client::builder()
199        .timeout(config.timeout)
200        .user_agent(&config.user_agent)
201        .pool_idle_timeout(config.pool_idle_timeout)
202        .pool_max_idle_per_host(config.pool_max_idle_per_host);
203
204    if let Some(ref proxy_url) = config.proxy {
205        let proxy = reqwest::Proxy::all(proxy_url)
206            // MED-001 fix: Redact credentials from error message
207            .map_err(|e| {
208                HttpError::InvalidProxy(format!("{}: {}", redact_proxy_url(proxy_url), e))
209            })?;
210        builder = builder.proxy(proxy);
211    }
212
213    // MED-003 fix: Provide detailed context on build failure
214    builder.build().map_err(|e| {
215        HttpError::BuildError(format!(
216            "Failed to build HTTP client (timeout: {:?}, pool_idle: {:?}, pool_max_idle: {}, proxy: {}): {}",
217            config.timeout,
218            config.pool_idle_timeout,
219            config.pool_max_idle_per_host,
220            config.proxy.as_ref().map_or_else(|| "none".to_string(), |p| redact_proxy_url(p)),
221            e
222        ))
223    })
224}
225
226/// Build a reqwest Client with default configuration
227pub fn build_default_client() -> Result<Client, HttpError> {
228    build_client(&HttpClientConfig::default())
229}
230
231/// Build a reqwest Client with just a proxy URL
232pub fn build_client_with_proxy(proxy: Option<&str>) -> Result<Client, HttpError> {
233    let config = HttpClientConfig::default().with_optional_proxy(proxy.map(String::from));
234    build_client(&config)
235}
236
237#[cfg(test)]
238mod tests {
239    use super::*;
240
241    #[test]
242    fn test_default_config() {
243        let config = HttpClientConfig::default();
244        assert_eq!(config.timeout, DEFAULT_TIMEOUT);
245        assert_eq!(config.user_agent, DEFAULT_USER_AGENT);
246        assert!(config.proxy.is_none());
247    }
248
249    #[test]
250    fn test_config_builder() {
251        let config = HttpClientConfig::new()
252            .with_timeout_secs(60)
253            .with_user_agent("CustomAgent/1.0")
254            .with_proxy("http://proxy:8080");
255
256        assert_eq!(config.timeout, Duration::from_secs(60));
257        assert_eq!(config.user_agent, "CustomAgent/1.0");
258        assert_eq!(config.proxy, Some("http://proxy:8080".to_string()));
259    }
260
261    #[test]
262    fn test_build_default_client() {
263        let client = build_default_client();
264        assert!(client.is_ok());
265    }
266
267    #[test]
268    fn test_build_client_with_config() {
269        let config = HttpClientConfig::new().with_timeout_secs(45);
270        let client = build_client(&config);
271        assert!(client.is_ok());
272    }
273
274    #[test]
275    fn test_proxy_url_format() {
276        // Valid proxy URLs work
277        let config = HttpClientConfig::new().with_proxy("http://proxy.example.com:8080");
278        let result = build_client(&config);
279        assert!(result.is_ok());
280
281        // Note: reqwest is lenient with proxy URL formats - most strings are accepted
282        // and errors only occur when the proxy is actually used for a connection.
283    }
284}
yldfi_common/http.rs

yldfi_common/
http.rs
