finance_query_core/client/
fetch_client.rs

1//! HTTP fetch client for Yahoo Finance requests.
2//!
3//! This module provides a low-level HTTP client with proxy support,
4//! cookie management, and timeout handling.
5
6use crate::client::error::YahooError;
7use reqwest::{cookie::Jar, Client, ClientBuilder};
8use std::io::Read;
9use std::sync::Arc;
10use std::time::Duration;
11
12const DEFAULT_TIMEOUT: Duration = Duration::from_secs(10);
13
14/// HTTP client for fetching data from Yahoo Finance.
15///
16/// Provides cookie management, proxy support, and various fetch methods
17/// for different content types (HTML, JSON).
18pub struct FetchClient {
19    client: Client,
20    cookie_jar: Arc<Jar>,
21    #[allow(dead_code)]
22    proxy: Option<String>,
23    /// Separate proxy URL only for Yahoo auth requests (to save bandwidth)
24    auth_proxy: Option<String>,
25}
26
27impl FetchClient {
28    /// Create a new FetchClient with optional proxy support.
29    ///
30    /// # Arguments
31    /// * `proxy` - Optional proxy URL for general requests
32    ///
33    /// # Environment Variables
34    /// * `AUTH_PROXY_URL` - If set, used for authentication requests instead of the general proxy
35    pub fn new(proxy: Option<String>) -> Result<Self, YahooError> {
36        // Check for auth-only proxy first, fall back to general proxy
37        let auth_proxy = std::env::var("AUTH_PROXY_URL").ok().or_else(|| proxy.clone());
38
39        let cookie_jar = Arc::new(Jar::default());
40
41        let mut builder = ClientBuilder::new()
42            .timeout(DEFAULT_TIMEOUT)
43            .cookie_store(true)
44            .cookie_provider(cookie_jar.clone())
45            .user_agent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36");
46
47        // Only use proxy for general client if PROXY_URL is set (not AUTH_PROXY_URL)
48        if let Some(proxy_url) = &proxy {
49            builder = builder.proxy(
50                reqwest::Proxy::all(proxy_url).map_err(|e| YahooError::NetworkError(e))?,
51            );
52        }
53
54        let client = builder.build().map_err(YahooError::NetworkError)?;
55
56        Ok(Self {
57            client,
58            cookie_jar,
59            proxy,
60            auth_proxy,
61        })
62    }
63
64
65    /// Get the proxy URL to use for auth requests only.
66    pub fn auth_proxy(&self) -> Option<&String> {
67        self.auth_proxy.as_ref()
68    }
69
70    /// Get a reference to the underlying reqwest Client.
71    pub fn client(&self) -> &Client {
72        &self.client
73    }
74
75    /// Get a reference to the cookie jar.
76    pub fn cookie_jar(&self) -> &Arc<Jar> {
77        &self.cookie_jar
78    }
79
80    /// Fetch a URL and return the response body as a string.
81    pub async fn fetch(&self, url: &str) -> Result<String, YahooError> {
82        self.fetch_with_timeout(url, DEFAULT_TIMEOUT).await
83    }
84
85    /// Fetch a URL expecting JSON response with proper Accept header.
86    pub async fn fetch_json(&self, url: &str) -> Result<String, YahooError> {
87        self.fetch_json_with_timeout(url, DEFAULT_TIMEOUT).await
88    }
89
90    /// Fetch a URL expecting JSON response with timeout and proper Accept header.
91    /// Note: Accept-Encoding is not set to avoid compression issues with JSON parsing.
92    pub async fn fetch_json_with_timeout(
93        &self,
94        url: &str,
95        timeout: Duration,
96    ) -> Result<String, YahooError> {
97        let response = match tokio::time::timeout(
98            timeout,
99            self.client
100                .get(url)
101                .timeout(timeout)
102                .header("Accept", "application/json")
103                .header("Accept-Language", "en-US,en;q=0.9")
104                // Don't request compression for JSON - it's usually small and compression can cause parsing issues
105                .header(
106                    "sec-ch-ua",
107                    r#""Chromium";v="122", "Google Chrome";v="122""#,
108                )
109                .header("sec-ch-ua-mobile", "?0")
110                .header("sec-ch-ua-platform", r#""Windows""#)
111                .send(),
112        )
113        .await
114        {
115            Ok(Ok(resp)) => resp,
116            Ok(Err(e)) => return Err(YahooError::NetworkError(e)),
117            Err(_) => {
118                return Err(YahooError::ParseError(format!(
119                    "Request to {} timed out after {:?}",
120                    url, timeout
121                )));
122            }
123        };
124
125        let status = response.status();
126        if !status.is_success() {
127            return Err(YahooError::HttpError(
128                status.as_u16(),
129                format!(
130                    "HTTP {}: {}",
131                    status,
132                    response.status().canonical_reason().unwrap_or("Unknown")
133                ),
134            ));
135        }
136
137        // Check Content-Encoding header to see if response is compressed
138        let content_encoding = response
139            .headers()
140            .get("content-encoding")
141            .and_then(|h| h.to_str().ok())
142            .unwrap_or("")
143            .to_lowercase();
144
145        let bytes = response.bytes().await.map_err(YahooError::NetworkError)?;
146
147        // If response is compressed, decompress it
148        let text = if content_encoding.contains("gzip") || content_encoding.contains("deflate") {
149            // Try to decompress gzip/deflate
150            let mut decoder = flate2::read::GzDecoder::new(&bytes[..]);
151            let mut decompressed = String::new();
152            decoder.read_to_string(&mut decompressed).map_err(|e| {
153                YahooError::ParseError(format!("Failed to decompress gzip response: {}", e))
154            })?;
155            decompressed
156        } else if content_encoding.contains("br") {
157            // Brotli compression - reqwest should handle this automatically, but if not, return error
158            return Err(YahooError::ParseError(
159                "Brotli compression detected but not automatically decompressed. This should not happen.".to_string()
160            ));
161        } else {
162            // Try to convert bytes to string
163            match String::from_utf8(bytes.to_vec()) {
164                Ok(text) => text,
165                Err(_) => {
166                    // If not valid UTF-8, might be compressed without Content-Encoding header
167                    // Try to decompress as gzip
168                    let mut decoder = flate2::read::GzDecoder::new(&bytes[..]);
169                    let mut decompressed = String::new();
170                    match decoder.read_to_string(&mut decompressed) {
171                        Ok(_) => decompressed,
172                        Err(_) => {
173                            // Not gzip either, return original error
174                            return Err(YahooError::ParseError(format!(
175                                "Response is not valid UTF-8 and not gzip compressed (length: {} bytes)",
176                                bytes.len()
177                            )));
178                        }
179                    }
180                }
181            }
182        };
183
184        Ok(text)
185    }
186
187
188    /// Fetch a URL with a custom timeout.
189    pub async fn fetch_with_timeout(&self, url: &str, timeout: Duration) -> Result<String, YahooError> {
190        // Create a request builder with timeout override
191        // Use tokio::time::timeout to ensure the request doesn't exceed the specified timeout
192        let response = match tokio::time::timeout(
193            timeout,
194            self.client
195                .get(url)
196                .timeout(timeout) // Explicitly set timeout on the request
197                .header(
198                    "Accept",
199                    "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
200                )
201                .header("Accept-Language", "en-US,en;q=0.9")
202                .header("Accept-Encoding", "gzip, deflate, br")
203                .header(
204                    "sec-ch-ua",
205                    r#""Chromium";v="122", "Google Chrome";v="122""#,
206                )
207                .header("sec-ch-ua-mobile", "?0")
208                .header("sec-ch-ua-platform", r#""Windows""#)
209                .send(),
210        )
211        .await
212        {
213            Ok(Ok(resp)) => resp,
214            Ok(Err(e)) => return Err(YahooError::NetworkError(e)),
215            Err(_) => {
216                // Timeout occurred - return a parse error with timeout message
217                return Err(YahooError::ParseError(format!(
218                    "Request to {} timed out after {:?}",
219                    url, timeout
220                )));
221            }
222        };
223
224        let status = response.status();
225        if !status.is_success() {
226            return Err(YahooError::HttpError(
227                status.as_u16(),
228                format!(
229                    "HTTP {}: {}",
230                    status,
231                    response.status().canonical_reason().unwrap_or("Unknown")
232                ),
233            ));
234        }
235
236        response.text().await.map_err(YahooError::NetworkError)
237    }
238
239    /// Fetch a URL and return the raw response.
240    pub async fn fetch_response(&self, url: &str) -> Result<reqwest::Response, YahooError> {
241        let response = self
242            .client
243            .get(url)
244            .header(
245                "Accept",
246                "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
247            )
248            .header("Accept-Language", "en-US,en;q=0.9")
249            .header("Accept-Encoding", "gzip, deflate, br")
250            .header(
251                "sec-ch-ua",
252                r#""Chromium";v="122", "Google Chrome";v="122""#,
253            )
254            .header("sec-ch-ua-mobile", "?0")
255            .header("sec-ch-ua-platform", r#""Windows""#)
256            .send()
257            .await
258            .map_err(YahooError::NetworkError)?;
259
260        Ok(response)
261    }
262}
finance_query_core/client/fetch_client.rs

finance_query_core/client/
fetch_client.rs
