finance_query_core/client/
fetch_client.rs

1//! HTTP fetch client for Yahoo Finance requests.
2//!
3//! This module provides a low-level HTTP client with proxy support,
4//! cookie management, and timeout handling.
5
6use crate::client::error::YahooError;
7use reqwest::{cookie::Jar, Client, ClientBuilder};
8use std::io::Read;
9use std::sync::Arc;
10use std::time::Duration;
11
12const DEFAULT_TIMEOUT: Duration = Duration::from_secs(10);
13
14/// HTTP client for fetching data from Yahoo Finance.
15///
16/// Provides cookie management, proxy support, and various fetch methods
17/// for different content types (HTML, JSON).
18#[derive(Debug)]
19pub struct FetchClient {
20    client: Client,
21    cookie_jar: Arc<Jar>,
22    #[allow(dead_code)]
23    proxy: Option<String>,
24    /// Separate proxy URL only for Yahoo auth requests (to save bandwidth)
25    auth_proxy: Option<String>,
26}
27
28impl FetchClient {
29    /// Create a new FetchClient with optional proxy support.
30    ///
31    /// # Arguments
32    /// * `proxy` - Optional proxy URL for general requests
33    ///
34    /// # Environment Variables
35    /// * `AUTH_PROXY_URL` - If set, used for authentication requests instead of the general proxy
36    pub fn new(proxy: Option<String>) -> Result<Self, YahooError> {
37        // Check for auth-only proxy first, fall back to general proxy
38        let auth_proxy = std::env::var("AUTH_PROXY_URL")
39            .ok()
40            .or_else(|| proxy.clone());
41
42        let cookie_jar = Arc::new(Jar::default());
43
44        let mut builder = ClientBuilder::new()
45            .timeout(DEFAULT_TIMEOUT)
46            .cookie_store(true)
47            .cookie_provider(cookie_jar.clone())
48            .user_agent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36");
49
50        // Only use proxy for general client if PROXY_URL is set (not AUTH_PROXY_URL)
51        if let Some(proxy_url) = &proxy {
52            builder =
53                builder.proxy(reqwest::Proxy::all(proxy_url).map_err(YahooError::NetworkError)?);
54        }
55
56        let client = builder.build().map_err(YahooError::NetworkError)?;
57
58        Ok(Self {
59            client,
60            cookie_jar,
61            proxy,
62            auth_proxy,
63        })
64    }
65
66    /// Get the proxy URL to use for auth requests only.
67    pub fn auth_proxy(&self) -> Option<&String> {
68        self.auth_proxy.as_ref()
69    }
70
71    /// Get a reference to the underlying reqwest Client.
72    pub fn client(&self) -> &Client {
73        &self.client
74    }
75
76    /// Get a reference to the cookie jar.
77    pub fn cookie_jar(&self) -> &Arc<Jar> {
78        &self.cookie_jar
79    }
80
81    /// Fetch a URL and return the response body as a string.
82    pub async fn fetch(&self, url: &str) -> Result<String, YahooError> {
83        self.fetch_with_timeout(url, DEFAULT_TIMEOUT).await
84    }
85
86    /// Fetch a URL expecting JSON response with proper Accept header.
87    pub async fn fetch_json(&self, url: &str) -> Result<String, YahooError> {
88        self.fetch_json_with_timeout(url, DEFAULT_TIMEOUT).await
89    }
90
91    /// Fetch a URL expecting JSON response with timeout and proper Accept header.
92    /// Note: Accept-Encoding is not set to avoid compression issues with JSON parsing.
93    pub async fn fetch_json_with_timeout(
94        &self,
95        url: &str,
96        timeout: Duration,
97    ) -> Result<String, YahooError> {
98        let response = match tokio::time::timeout(
99            timeout,
100            self.client
101                .get(url)
102                .timeout(timeout)
103                .header("Accept", "application/json")
104                .header("Accept-Language", "en-US,en;q=0.9")
105                // Don't request compression for JSON - it's usually small and compression can cause parsing issues
106                .header(
107                    "sec-ch-ua",
108                    r#""Chromium";v="122", "Google Chrome";v="122""#,
109                )
110                .header("sec-ch-ua-mobile", "?0")
111                .header("sec-ch-ua-platform", r#""Windows""#)
112                .send(),
113        )
114        .await
115        {
116            Ok(Ok(resp)) => resp,
117            Ok(Err(e)) => return Err(YahooError::NetworkError(e)),
118            Err(_) => {
119                return Err(YahooError::ParseError(format!(
120                    "Request to {} timed out after {:?}",
121                    url, timeout
122                )));
123            }
124        };
125
126        let status = response.status();
127        if !status.is_success() {
128            return Err(YahooError::HttpError(
129                status.as_u16(),
130                format!(
131                    "HTTP {}: {}",
132                    status,
133                    response.status().canonical_reason().unwrap_or("Unknown")
134                ),
135            ));
136        }
137
138        // Check Content-Encoding header to see if response is compressed
139        let content_encoding = response
140            .headers()
141            .get("content-encoding")
142            .and_then(|h| h.to_str().ok())
143            .unwrap_or("")
144            .to_lowercase();
145
146        let bytes = response.bytes().await.map_err(YahooError::NetworkError)?;
147
148        // If response is compressed, decompress it
149        let text = if content_encoding.contains("gzip") || content_encoding.contains("deflate") {
150            // Try to decompress gzip/deflate
151            let mut decoder = flate2::read::GzDecoder::new(&bytes[..]);
152            let mut decompressed = String::new();
153            decoder.read_to_string(&mut decompressed).map_err(|e| {
154                YahooError::ParseError(format!("Failed to decompress gzip response: {}", e))
155            })?;
156            decompressed
157        } else if content_encoding.contains("br") {
158            // Brotli compression - reqwest should handle this automatically, but if not, return error
159            return Err(YahooError::ParseError(
160                "Brotli compression detected but not automatically decompressed. This should not happen.".to_string()
161            ));
162        } else {
163            // Try to convert bytes to string
164            match String::from_utf8(bytes.to_vec()) {
165                Ok(text) => text,
166                Err(_) => {
167                    // If not valid UTF-8, might be compressed without Content-Encoding header
168                    // Try to decompress as gzip
169                    let mut decoder = flate2::read::GzDecoder::new(&bytes[..]);
170                    let mut decompressed = String::new();
171                    match decoder.read_to_string(&mut decompressed) {
172                        Ok(_) => decompressed,
173                        Err(_) => {
174                            // Not gzip either, return original error
175                            return Err(YahooError::ParseError(format!(
176                                "Response is not valid UTF-8 and not gzip compressed (length: {} bytes)",
177                                bytes.len()
178                            )));
179                        }
180                    }
181                }
182            }
183        };
184
185        Ok(text)
186    }
187
188    /// Fetch a URL with a custom timeout.
189    pub async fn fetch_with_timeout(
190        &self,
191        url: &str,
192        timeout: Duration,
193    ) -> Result<String, YahooError> {
194        // Create a request builder with timeout override
195        // Use tokio::time::timeout to ensure the request doesn't exceed the specified timeout
196        let response = match tokio::time::timeout(
197            timeout,
198            self.client
199                .get(url)
200                .timeout(timeout) // Explicitly set timeout on the request
201                .header(
202                    "Accept",
203                    "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
204                )
205                .header("Accept-Language", "en-US,en;q=0.9")
206                .header("Accept-Encoding", "gzip, deflate, br")
207                .header(
208                    "sec-ch-ua",
209                    r#""Chromium";v="122", "Google Chrome";v="122""#,
210                )
211                .header("sec-ch-ua-mobile", "?0")
212                .header("sec-ch-ua-platform", r#""Windows""#)
213                .send(),
214        )
215        .await
216        {
217            Ok(Ok(resp)) => resp,
218            Ok(Err(e)) => return Err(YahooError::NetworkError(e)),
219            Err(_) => {
220                // Timeout occurred - return a parse error with timeout message
221                return Err(YahooError::ParseError(format!(
222                    "Request to {} timed out after {:?}",
223                    url, timeout
224                )));
225            }
226        };
227
228        let status = response.status();
229        if !status.is_success() {
230            return Err(YahooError::HttpError(
231                status.as_u16(),
232                format!(
233                    "HTTP {}: {}",
234                    status,
235                    response.status().canonical_reason().unwrap_or("Unknown")
236                ),
237            ));
238        }
239
240        response.text().await.map_err(YahooError::NetworkError)
241    }
242
243    /// Fetch a URL and return the raw response.
244    pub async fn fetch_response(&self, url: &str) -> Result<reqwest::Response, YahooError> {
245        let response = self
246            .client
247            .get(url)
248            .header(
249                "Accept",
250                "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
251            )
252            .header("Accept-Language", "en-US,en;q=0.9")
253            .header("Accept-Encoding", "gzip, deflate, br")
254            .header(
255                "sec-ch-ua",
256                r#""Chromium";v="122", "Google Chrome";v="122""#,
257            )
258            .header("sec-ch-ua-mobile", "?0")
259            .header("sec-ch-ua-platform", r#""Windows""#)
260            .send()
261            .await
262            .map_err(YahooError::NetworkError)?;
263
264        Ok(response)
265    }
266}
finance_query_core/client/fetch_client.rs

finance_query_core/client/
fetch_client.rs
