periplon_sdk/
data_fetcher.rs

1//! Data fetching module for APIs and files
2//!
3//! Provides unified interface for fetching data from various sources:
4//! - HTTP/HTTPS APIs (GET, POST, etc.)
5//! - Local file system (text, JSON, binary)
6//! - Async operations with error handling
7
8use serde::{Deserialize, Serialize};
9use serde_json::Value;
10use std::collections::HashMap;
11use std::path::Path;
12use tokio::fs;
13
14/// Errors that can occur during data fetching
15#[derive(Debug, thiserror::Error)]
16pub enum FetchError {
17    #[error("HTTP request failed: {0}")]
18    HttpError(String),
19
20    #[error("File I/O error: {0}")]
21    IoError(#[from] std::io::Error),
22
23    #[error("JSON parsing error: {0}")]
24    JsonError(#[from] serde_json::Error),
25
26    #[error("Invalid URL: {0}")]
27    InvalidUrl(String),
28
29    #[error("Network error: {0}")]
30    NetworkError(String),
31}
32
33/// HTTP methods supported
34#[derive(Debug, Clone, Copy)]
35pub enum HttpMethod {
36    Get,
37    Post,
38    Put,
39    Delete,
40    Patch,
41}
42
43/// HTTP request configuration
44#[derive(Debug, Clone)]
45pub struct HttpRequest {
46    pub url: String,
47    pub method: HttpMethod,
48    pub headers: HashMap<String, String>,
49    pub body: Option<String>,
50    pub timeout_secs: u64,
51}
52
53impl HttpRequest {
54    pub fn new(url: impl Into<String>) -> Self {
55        Self {
56            url: url.into(),
57            method: HttpMethod::Get,
58            headers: HashMap::new(),
59            body: None,
60            timeout_secs: 30,
61        }
62    }
63
64    pub fn method(mut self, method: HttpMethod) -> Self {
65        self.method = method;
66        self
67    }
68
69    pub fn header(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
70        self.headers.insert(key.into(), value.into());
71        self
72    }
73
74    pub fn body(mut self, body: impl Into<String>) -> Self {
75        self.body = Some(body.into());
76        self
77    }
78
79    pub fn json_body<T: Serialize>(mut self, data: &T) -> Result<Self, FetchError> {
80        self.body = Some(serde_json::to_string(data)?);
81        self.headers
82            .insert("Content-Type".to_string(), "application/json".to_string());
83        Ok(self)
84    }
85
86    pub fn timeout(mut self, secs: u64) -> Self {
87        self.timeout_secs = secs;
88        self
89    }
90}
91
92/// HTTP response
93#[derive(Debug, Clone)]
94pub struct HttpResponse {
95    pub status: u16,
96    pub headers: HashMap<String, String>,
97    pub body: String,
98}
99
100impl HttpResponse {
101    /// Parse response body as JSON
102    pub fn json<T: for<'de> Deserialize<'de>>(&self) -> Result<T, FetchError> {
103        Ok(serde_json::from_str(&self.body)?)
104    }
105
106    /// Parse response body as dynamic JSON value
107    pub fn json_value(&self) -> Result<Value, FetchError> {
108        Ok(serde_json::from_str(&self.body)?)
109    }
110
111    /// Get response body as text
112    pub fn text(&self) -> &str {
113        &self.body
114    }
115
116    /// Check if response was successful (2xx status code)
117    pub fn is_success(&self) -> bool {
118        (200..300).contains(&self.status)
119    }
120}
121
122/// Main data fetcher for APIs and files
123pub struct DataFetcher {
124    user_agent: String,
125    default_headers: HashMap<String, String>,
126}
127
128impl Default for DataFetcher {
129    fn default() -> Self {
130        Self::new()
131    }
132}
133
134impl DataFetcher {
135    /// Create a new data fetcher with default settings
136    pub fn new() -> Self {
137        let mut default_headers = HashMap::new();
138        default_headers.insert("User-Agent".to_string(), "DataFetcher/1.0".to_string());
139
140        Self {
141            user_agent: "DataFetcher/1.0".to_string(),
142            default_headers,
143        }
144    }
145
146    /// Set custom user agent
147    pub fn user_agent(mut self, agent: impl Into<String>) -> Self {
148        self.user_agent = agent.into();
149        self.default_headers
150            .insert("User-Agent".to_string(), self.user_agent.clone());
151        self
152    }
153
154    /// Add default header for all requests
155    pub fn default_header(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
156        self.default_headers.insert(key.into(), value.into());
157        self
158    }
159
160    /// Fetch data from HTTP API
161    pub async fn fetch_http(&self, request: HttpRequest) -> Result<HttpResponse, FetchError> {
162        // In a real implementation, use reqwest or hyper
163        // For now, provide a mock implementation that demonstrates the interface
164
165        // Validate URL
166        if !request.url.starts_with("http://") && !request.url.starts_with("https://") {
167            return Err(FetchError::InvalidUrl(format!(
168                "URL must start with http:// or https://, got: {}",
169                request.url
170            )));
171        }
172
173        // Simulate network request (in production, replace with actual HTTP client)
174        tokio::time::sleep(tokio::time::Duration::from_millis(100)).await;
175
176        // Mock response
177        let response = HttpResponse {
178            status: 200,
179            headers: HashMap::from([("content-type".to_string(), "application/json".to_string())]),
180            body: r#"{"message": "Mock API response", "status": "success"}"#.to_string(),
181        };
182
183        Ok(response)
184    }
185
186    /// Simple GET request
187    pub async fn get(&self, url: impl Into<String>) -> Result<HttpResponse, FetchError> {
188        self.fetch_http(HttpRequest::new(url)).await
189    }
190
191    /// Simple POST request with JSON body
192    pub async fn post_json<T: Serialize>(
193        &self,
194        url: impl Into<String>,
195        body: &T,
196    ) -> Result<HttpResponse, FetchError> {
197        let request = HttpRequest::new(url)
198            .method(HttpMethod::Post)
199            .json_body(body)?;
200        self.fetch_http(request).await
201    }
202
203    /// Read text file from filesystem
204    pub async fn read_text_file<P: AsRef<Path>>(&self, path: P) -> Result<String, FetchError> {
205        Ok(fs::read_to_string(path).await?)
206    }
207
208    /// Read binary file from filesystem
209    pub async fn read_binary_file<P: AsRef<Path>>(&self, path: P) -> Result<Vec<u8>, FetchError> {
210        Ok(fs::read(path).await?)
211    }
212
213    /// Read and parse JSON file
214    pub async fn read_json_file<P: AsRef<Path>, T: for<'de> Deserialize<'de>>(
215        &self,
216        path: P,
217    ) -> Result<T, FetchError> {
218        let content = self.read_text_file(path).await?;
219        Ok(serde_json::from_str(&content)?)
220    }
221
222    /// Read JSON file as dynamic value
223    pub async fn read_json_value<P: AsRef<Path>>(&self, path: P) -> Result<Value, FetchError> {
224        let content = self.read_text_file(path).await?;
225        Ok(serde_json::from_str(&content)?)
226    }
227
228    /// Read file line by line
229    pub async fn read_lines<P: AsRef<Path>>(&self, path: P) -> Result<Vec<String>, FetchError> {
230        let content = self.read_text_file(path).await?;
231        Ok(content.lines().map(|s| s.to_string()).collect())
232    }
233
234    /// Check if file exists
235    pub async fn file_exists<P: AsRef<Path>>(&self, path: P) -> bool {
236        fs::metadata(path).await.is_ok()
237    }
238
239    /// Get file metadata (size, modified time, etc.)
240    pub async fn file_metadata<P: AsRef<Path>>(&self, path: P) -> Result<FileMetadata, FetchError> {
241        let metadata = fs::metadata(path).await?;
242        Ok(FileMetadata {
243            size: metadata.len(),
244            is_file: metadata.is_file(),
245            is_dir: metadata.is_dir(),
246            read_only: metadata.permissions().readonly(),
247        })
248    }
249}
250
251/// File metadata information
252#[derive(Debug, Clone, Serialize, Deserialize)]
253pub struct FileMetadata {
254    pub size: u64,
255    pub is_file: bool,
256    pub is_dir: bool,
257    pub read_only: bool,
258}
259
260/// Convenience functions for quick operations
261pub mod quick {
262    use super::*;
263
264    /// Quick GET request
265    pub async fn get(url: impl Into<String>) -> Result<HttpResponse, FetchError> {
266        DataFetcher::new().get(url).await
267    }
268
269    /// Quick POST with JSON
270    pub async fn post_json<T: Serialize>(
271        url: impl Into<String>,
272        body: &T,
273    ) -> Result<HttpResponse, FetchError> {
274        DataFetcher::new().post_json(url, body).await
275    }
276
277    /// Quick file read
278    pub async fn read_file<P: AsRef<Path>>(path: P) -> Result<String, FetchError> {
279        DataFetcher::new().read_text_file(path).await
280    }
281
282    /// Quick JSON file read
283    pub async fn read_json<P: AsRef<Path>, T: for<'de> Deserialize<'de>>(
284        path: P,
285    ) -> Result<T, FetchError> {
286        DataFetcher::new().read_json_file(path).await
287    }
288}
289
290#[cfg(test)]
291mod tests {
292    use super::*;
293    use std::io::Write;
294    use tempfile::NamedTempFile;
295
296    #[tokio::test]
297    async fn test_http_request_builder() {
298        let request = HttpRequest::new("https://api.example.com/data")
299            .method(HttpMethod::Post)
300            .header("Authorization", "Bearer token123")
301            .body(r#"{"key": "value"}"#)
302            .timeout(60);
303
304        assert_eq!(request.url, "https://api.example.com/data");
305        assert!(matches!(request.method, HttpMethod::Post));
306        assert_eq!(
307            request.headers.get("Authorization").unwrap(),
308            "Bearer token123"
309        );
310        assert_eq!(request.timeout_secs, 60);
311    }
312
313    #[tokio::test]
314    async fn test_json_body() {
315        #[derive(Serialize)]
316        struct TestData {
317            name: String,
318            value: i32,
319        }
320
321        let data = TestData {
322            name: "test".to_string(),
323            value: 42,
324        };
325
326        let request = HttpRequest::new("https://api.example.com/data")
327            .json_body(&data)
328            .unwrap();
329
330        assert!(request.body.is_some());
331        assert_eq!(
332            request.headers.get("Content-Type").unwrap(),
333            "application/json"
334        );
335    }
336
337    #[tokio::test]
338    async fn test_mock_http_fetch() {
339        let fetcher = DataFetcher::new();
340        let response = fetcher.get("https://api.example.com/test").await.unwrap();
341
342        assert!(response.is_success());
343        assert_eq!(response.status, 200);
344    }
345
346    #[tokio::test]
347    async fn test_invalid_url() {
348        let fetcher = DataFetcher::new();
349        let result = fetcher.get("not-a-valid-url").await;
350
351        assert!(result.is_err());
352        assert!(matches!(result.unwrap_err(), FetchError::InvalidUrl(_)));
353    }
354
355    #[tokio::test]
356    async fn test_read_text_file() {
357        let mut temp_file = NamedTempFile::new().unwrap();
358        writeln!(temp_file, "Hello, World!").unwrap();
359        writeln!(temp_file, "Second line").unwrap();
360
361        let fetcher = DataFetcher::new();
362        let content = fetcher.read_text_file(temp_file.path()).await.unwrap();
363
364        assert!(content.contains("Hello, World!"));
365        assert!(content.contains("Second line"));
366    }
367
368    #[tokio::test]
369    async fn test_read_json_file() {
370        let mut temp_file = NamedTempFile::new().unwrap();
371        writeln!(temp_file, r#"{{"name": "test", "value": 42}}"#).unwrap();
372
373        let fetcher = DataFetcher::new();
374        let json: Value = fetcher.read_json_value(temp_file.path()).await.unwrap();
375
376        assert_eq!(json["name"], "test");
377        assert_eq!(json["value"], 42);
378    }
379
380    #[tokio::test]
381    async fn test_read_lines() {
382        let mut temp_file = NamedTempFile::new().unwrap();
383        writeln!(temp_file, "Line 1").unwrap();
384        writeln!(temp_file, "Line 2").unwrap();
385        writeln!(temp_file, "Line 3").unwrap();
386
387        let fetcher = DataFetcher::new();
388        let lines = fetcher.read_lines(temp_file.path()).await.unwrap();
389
390        assert_eq!(lines.len(), 3);
391        assert_eq!(lines[0], "Line 1");
392        assert_eq!(lines[1], "Line 2");
393        assert_eq!(lines[2], "Line 3");
394    }
395
396    #[tokio::test]
397    async fn test_file_exists() {
398        let temp_file = NamedTempFile::new().unwrap();
399
400        let fetcher = DataFetcher::new();
401        assert!(fetcher.file_exists(temp_file.path()).await);
402        assert!(!fetcher.file_exists("/nonexistent/path/file.txt").await);
403    }
404
405    #[tokio::test]
406    async fn test_file_metadata() {
407        let mut temp_file = NamedTempFile::new().unwrap();
408        writeln!(temp_file, "Test content").unwrap();
409
410        let fetcher = DataFetcher::new();
411        let metadata = fetcher.file_metadata(temp_file.path()).await.unwrap();
412
413        assert!(metadata.is_file);
414        assert!(!metadata.is_dir);
415        assert!(metadata.size > 0);
416    }
417
418    #[tokio::test]
419    async fn test_quick_functions() {
420        let mut temp_file = NamedTempFile::new().unwrap();
421        writeln!(temp_file, "Quick read test").unwrap();
422
423        let content = quick::read_file(temp_file.path()).await.unwrap();
424        assert!(content.contains("Quick read test"));
425    }
426}
periplon_sdk/data_fetcher.rs

periplon_sdk/
data_fetcher.rs
