finance_query_core/client/
fetch_client.rs

1//! HTTP fetch client for Yahoo Finance requests.
2//!
3//! This module provides a low-level HTTP client with proxy support,
4//! cookie management, and timeout handling.
5
6use crate::client::error::YahooError;
7use reqwest::{cookie::Jar, Client, ClientBuilder};
8use std::io::Read;
9use std::sync::Arc;
10use std::time::Duration;
11
12const DEFAULT_TIMEOUT: Duration = Duration::from_secs(10);
13
14/// HTTP client for fetching data from Yahoo Finance.
15///
16/// Provides cookie management, proxy support, and various fetch methods
17/// for different content types (HTML, JSON).
18pub struct FetchClient {
19    client: Client,
20    cookie_jar: Arc<Jar>,
21    #[allow(dead_code)]
22    proxy: Option<String>,
23    /// Separate proxy URL only for Yahoo auth requests (to save bandwidth)
24    auth_proxy: Option<String>,
25}
26
27impl FetchClient {
28    /// Create a new FetchClient with optional proxy support.
29    ///
30    /// # Arguments
31    /// * `proxy` - Optional proxy URL for general requests
32    ///
33    /// # Environment Variables
34    /// * `AUTH_PROXY_URL` - If set, used for authentication requests instead of the general proxy
35    pub fn new(proxy: Option<String>) -> Result<Self, YahooError> {
36        // Check for auth-only proxy first, fall back to general proxy
37        let auth_proxy = std::env::var("AUTH_PROXY_URL")
38            .ok()
39            .or_else(|| proxy.clone());
40
41        let cookie_jar = Arc::new(Jar::default());
42
43        let mut builder = ClientBuilder::new()
44            .timeout(DEFAULT_TIMEOUT)
45            .cookie_store(true)
46            .cookie_provider(cookie_jar.clone())
47            .user_agent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36");
48
49        // Only use proxy for general client if PROXY_URL is set (not AUTH_PROXY_URL)
50        if let Some(proxy_url) = &proxy {
51            builder =
52                builder.proxy(reqwest::Proxy::all(proxy_url).map_err(YahooError::NetworkError)?);
53        }
54
55        let client = builder.build().map_err(YahooError::NetworkError)?;
56
57        Ok(Self {
58            client,
59            cookie_jar,
60            proxy,
61            auth_proxy,
62        })
63    }
64
65    /// Get the proxy URL to use for auth requests only.
66    pub fn auth_proxy(&self) -> Option<&String> {
67        self.auth_proxy.as_ref()
68    }
69
70    /// Get a reference to the underlying reqwest Client.
71    pub fn client(&self) -> &Client {
72        &self.client
73    }
74
75    /// Get a reference to the cookie jar.
76    pub fn cookie_jar(&self) -> &Arc<Jar> {
77        &self.cookie_jar
78    }
79
80    /// Fetch a URL and return the response body as a string.
81    pub async fn fetch(&self, url: &str) -> Result<String, YahooError> {
82        self.fetch_with_timeout(url, DEFAULT_TIMEOUT).await
83    }
84
85    /// Fetch a URL expecting JSON response with proper Accept header.
86    pub async fn fetch_json(&self, url: &str) -> Result<String, YahooError> {
87        self.fetch_json_with_timeout(url, DEFAULT_TIMEOUT).await
88    }
89
90    /// Fetch a URL expecting JSON response with timeout and proper Accept header.
91    /// Note: Accept-Encoding is not set to avoid compression issues with JSON parsing.
92    pub async fn fetch_json_with_timeout(
93        &self,
94        url: &str,
95        timeout: Duration,
96    ) -> Result<String, YahooError> {
97        let response = match tokio::time::timeout(
98            timeout,
99            self.client
100                .get(url)
101                .timeout(timeout)
102                .header("Accept", "application/json")
103                .header("Accept-Language", "en-US,en;q=0.9")
104                // Don't request compression for JSON - it's usually small and compression can cause parsing issues
105                .header(
106                    "sec-ch-ua",
107                    r#""Chromium";v="122", "Google Chrome";v="122""#,
108                )
109                .header("sec-ch-ua-mobile", "?0")
110                .header("sec-ch-ua-platform", r#""Windows""#)
111                .send(),
112        )
113        .await
114        {
115            Ok(Ok(resp)) => resp,
116            Ok(Err(e)) => return Err(YahooError::NetworkError(e)),
117            Err(_) => {
118                return Err(YahooError::ParseError(format!(
119                    "Request to {} timed out after {:?}",
120                    url, timeout
121                )));
122            }
123        };
124
125        let status = response.status();
126        if !status.is_success() {
127            return Err(YahooError::HttpError(
128                status.as_u16(),
129                format!(
130                    "HTTP {}: {}",
131                    status,
132                    response.status().canonical_reason().unwrap_or("Unknown")
133                ),
134            ));
135        }
136
137        // Check Content-Encoding header to see if response is compressed
138        let content_encoding = response
139            .headers()
140            .get("content-encoding")
141            .and_then(|h| h.to_str().ok())
142            .unwrap_or("")
143            .to_lowercase();
144
145        let bytes = response.bytes().await.map_err(YahooError::NetworkError)?;
146
147        // If response is compressed, decompress it
148        let text = if content_encoding.contains("gzip") || content_encoding.contains("deflate") {
149            // Try to decompress gzip/deflate
150            let mut decoder = flate2::read::GzDecoder::new(&bytes[..]);
151            let mut decompressed = String::new();
152            decoder.read_to_string(&mut decompressed).map_err(|e| {
153                YahooError::ParseError(format!("Failed to decompress gzip response: {}", e))
154            })?;
155            decompressed
156        } else if content_encoding.contains("br") {
157            // Brotli compression - reqwest should handle this automatically, but if not, return error
158            return Err(YahooError::ParseError(
159                "Brotli compression detected but not automatically decompressed. This should not happen.".to_string()
160            ));
161        } else {
162            // Try to convert bytes to string
163            match String::from_utf8(bytes.to_vec()) {
164                Ok(text) => text,
165                Err(_) => {
166                    // If not valid UTF-8, might be compressed without Content-Encoding header
167                    // Try to decompress as gzip
168                    let mut decoder = flate2::read::GzDecoder::new(&bytes[..]);
169                    let mut decompressed = String::new();
170                    match decoder.read_to_string(&mut decompressed) {
171                        Ok(_) => decompressed,
172                        Err(_) => {
173                            // Not gzip either, return original error
174                            return Err(YahooError::ParseError(format!(
175                                "Response is not valid UTF-8 and not gzip compressed (length: {} bytes)",
176                                bytes.len()
177                            )));
178                        }
179                    }
180                }
181            }
182        };
183
184        Ok(text)
185    }
186
187    /// Fetch a URL with a custom timeout.
188    pub async fn fetch_with_timeout(
189        &self,
190        url: &str,
191        timeout: Duration,
192    ) -> Result<String, YahooError> {
193        // Create a request builder with timeout override
194        // Use tokio::time::timeout to ensure the request doesn't exceed the specified timeout
195        let response = match tokio::time::timeout(
196            timeout,
197            self.client
198                .get(url)
199                .timeout(timeout) // Explicitly set timeout on the request
200                .header(
201                    "Accept",
202                    "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
203                )
204                .header("Accept-Language", "en-US,en;q=0.9")
205                .header("Accept-Encoding", "gzip, deflate, br")
206                .header(
207                    "sec-ch-ua",
208                    r#""Chromium";v="122", "Google Chrome";v="122""#,
209                )
210                .header("sec-ch-ua-mobile", "?0")
211                .header("sec-ch-ua-platform", r#""Windows""#)
212                .send(),
213        )
214        .await
215        {
216            Ok(Ok(resp)) => resp,
217            Ok(Err(e)) => return Err(YahooError::NetworkError(e)),
218            Err(_) => {
219                // Timeout occurred - return a parse error with timeout message
220                return Err(YahooError::ParseError(format!(
221                    "Request to {} timed out after {:?}",
222                    url, timeout
223                )));
224            }
225        };
226
227        let status = response.status();
228        if !status.is_success() {
229            return Err(YahooError::HttpError(
230                status.as_u16(),
231                format!(
232                    "HTTP {}: {}",
233                    status,
234                    response.status().canonical_reason().unwrap_or("Unknown")
235                ),
236            ));
237        }
238
239        response.text().await.map_err(YahooError::NetworkError)
240    }
241
242    /// Fetch a URL and return the raw response.
243    pub async fn fetch_response(&self, url: &str) -> Result<reqwest::Response, YahooError> {
244        let response = self
245            .client
246            .get(url)
247            .header(
248                "Accept",
249                "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
250            )
251            .header("Accept-Language", "en-US,en;q=0.9")
252            .header("Accept-Encoding", "gzip, deflate, br")
253            .header(
254                "sec-ch-ua",
255                r#""Chromium";v="122", "Google Chrome";v="122""#,
256            )
257            .header("sec-ch-ua-mobile", "?0")
258            .header("sec-ch-ua-platform", r#""Windows""#)
259            .send()
260            .await
261            .map_err(YahooError::NetworkError)?;
262
263        Ok(response)
264    }
265}
finance_query_core/client/fetch_client.rs

finance_query_core/client/
fetch_client.rs
