scirs2_io/network/
mod.rs

1//! Network I/O and cloud storage integration
2//!
3//! This module provides functionality for reading and writing files over network protocols
4//! and integrating with cloud storage services. It supports efficient streaming, caching,
5//! and secure authentication for various cloud providers.
6//!
7//! ## Features
8//!
9//! - **HTTP/HTTPS I/O**: Download and upload files via HTTP protocols
10//! - **Cloud Storage**: Integration with AWS S3, Google Cloud Storage, Azure Blob Storage
11//! - **Streaming**: Efficient handling of large files with minimal memory usage
12//! - **Caching**: Local caching of remote files for offline access
13//! - **Authentication**: Secure handling of credentials and API keys
14//! - **Retry Logic**: Robust error handling with automatic retry mechanisms
15//!
16//! ## Examples
17//!
18//! ```rust,no_run
19//! use scirs2_io::network::NetworkClient;
20//!
21//! // Create a network client
22//! let client = NetworkClient::new();
23//! println!("Network client created for file operations");
24//! # Ok::<(), Box<dyn std::error::Error>>(())
25//! ```
26
27use crate::error::{IoError, Result};
28use std::collections::HashMap;
29use std::path::Path;
30use std::time::Duration;
31
32/// Cloud storage integration
33pub mod cloud;
34/// HTTP client functionality  
35pub mod http;
36/// Streaming I/O operations
37pub mod streaming;
38
39/// Network client configuration
40#[derive(Debug, Clone)]
41pub struct NetworkConfig {
42    /// Connection timeout in seconds
43    pub connect_timeout: Duration,
44    /// Read timeout in seconds
45    pub read_timeout: Duration,
46    /// Maximum number of retry attempts
47    pub max_retries: u32,
48    /// User agent string for HTTP requests
49    pub user_agent: String,
50    /// Custom HTTP headers
51    pub headers: HashMap<String, String>,
52    /// Enable response compression
53    pub compression: bool,
54    /// Local cache directory for downloaded files
55    pub cache_dir: Option<String>,
56    /// Maximum cache size in MB
57    pub max_cache_size: u64,
58}
59
60impl Default for NetworkConfig {
61    fn default() -> Self {
62        let mut headers = HashMap::new();
63        headers.insert("Accept".to_string(), "*/*".to_string());
64
65        Self {
66            connect_timeout: Duration::from_secs(30),
67            read_timeout: Duration::from_secs(300),
68            max_retries: 3,
69            user_agent: "scirs2-io/0.1.0".to_string(),
70            headers,
71            compression: true,
72            cache_dir: None,
73            max_cache_size: 1024, // 1GB default
74        }
75    }
76}
77
78/// Network client for remote I/O operations
79#[derive(Debug)]
80pub struct NetworkClient {
81    config: NetworkConfig,
82    #[cfg(feature = "reqwest")]
83    http_client: Option<reqwest::Client>,
84    cloud_provider: Option<cloud::CloudProvider>,
85}
86
87impl Default for NetworkClient {
88    fn default() -> Self {
89        Self::new()
90    }
91}
92
93impl NetworkClient {
94    /// Create a new network client with default configuration
95    pub fn new() -> Self {
96        Self {
97            config: NetworkConfig::default(),
98            #[cfg(feature = "reqwest")]
99            http_client: None,
100            cloud_provider: None,
101        }
102    }
103
104    /// Create a new network client with custom configuration
105    pub fn with_config(config: NetworkConfig) -> Self {
106        Self {
107            config,
108            #[cfg(feature = "reqwest")]
109            http_client: None,
110            cloud_provider: None,
111        }
112    }
113
114    /// Set cloud provider for cloud storage operations
115    pub fn with_cloud_provider(mut self, provider: cloud::CloudProvider) -> Self {
116        self.cloud_provider = Some(provider);
117        self
118    }
119
120    /// Set cache directory for downloaded files
121    pub fn with_cache_dir<P: AsRef<Path>>(mut self, cache_dir: P) -> Self {
122        self.config.cache_dir = Some(cache_dir.as_ref().to_string_lossy().to_string());
123        self
124    }
125
126    /// Download a file from URL to local path
127    #[cfg(feature = "reqwest")]
128    pub async fn download<P: AsRef<Path>>(&self, url: &str, localpath: P) -> Result<()> {
129        if let Some(_client) = &self.http_client {
130            // Create HttpClient with current config and use it for download
131            let mut http_client = http::HttpClient::new(self.config.clone());
132            http_client.init()?;
133            http_client.download(url, localpath).await
134        } else {
135            Err(IoError::ConfigError(
136                "HTTP client not configured".to_string(),
137            ))
138        }
139    }
140
141    /// Upload a file from local path to URL
142    #[cfg(feature = "reqwest")]
143    pub async fn upload<P: AsRef<Path>>(&self, localpath: P, url: &str) -> Result<()> {
144        if let Some(_client) = &self.http_client {
145            // Create HttpClient with current config and use it for upload
146            let mut http_client = http::HttpClient::new(self.config.clone());
147            http_client.init()?;
148            http_client.upload(localpath, url).await
149        } else {
150            Err(IoError::ConfigError(
151                "HTTP client not configured".to_string(),
152            ))
153        }
154    }
155
156    /// Download a file to cloud storage
157    pub async fn upload_to_cloud<P: AsRef<Path>>(
158        &self,
159        localpath: P,
160        remote_path: &str,
161    ) -> Result<()> {
162        if let Some(ref provider) = self.cloud_provider {
163            provider.upload_file(localpath, remote_path).await
164        } else {
165            Err(IoError::ConfigError(
166                "No cloud provider configured".to_string(),
167            ))
168        }
169    }
170
171    /// Download a file from cloud storage
172    pub async fn download_from_cloud<P: AsRef<Path>>(
173        &self,
174        remote_path: &str,
175        localpath: P,
176    ) -> Result<()> {
177        if let Some(ref provider) = self.cloud_provider {
178            provider.download_file(remote_path, localpath).await
179        } else {
180            Err(IoError::ConfigError(
181                "No cloud provider configured".to_string(),
182            ))
183        }
184    }
185
186    /// List files in cloud storage path
187    pub async fn list_cloud_files(&self, path: &str) -> Result<Vec<String>> {
188        if let Some(ref provider) = self.cloud_provider {
189            provider.list_files(path).await
190        } else {
191            Err(IoError::ConfigError(
192                "No cloud provider configured".to_string(),
193            ))
194        }
195    }
196
197    /// Check if a file exists in cloud storage
198    pub async fn cloud_file_exists(&self, path: &str) -> Result<bool> {
199        if let Some(ref provider) = self.cloud_provider {
200            provider.file_exists(path).await
201        } else {
202            Err(IoError::ConfigError(
203                "No cloud provider configured".to_string(),
204            ))
205        }
206    }
207
208    /// Get file metadata from cloud storage
209    pub async fn get_cloud_file_metadata(&self, path: &str) -> Result<cloud::FileMetadata> {
210        if let Some(ref provider) = self.cloud_provider {
211            provider.get_metadata(path).await
212        } else {
213            Err(IoError::ConfigError(
214                "No cloud provider configured".to_string(),
215            ))
216        }
217    }
218
219    /// Clear local cache
220    pub fn clear_cache(&self) -> Result<()> {
221        if let Some(ref cache_dir) = self.config.cache_dir {
222            let cache_path = Path::new(cache_dir);
223            if cache_path.exists() {
224                std::fs::remove_dir_all(cache_path)
225                    .map_err(|e| IoError::FileError(format!("Failed to clear cache: {}", e)))?;
226                std::fs::create_dir_all(cache_path).map_err(|e| {
227                    IoError::FileError(format!("Failed to recreate cache dir: {}", e))
228                })?;
229            }
230        }
231        Ok(())
232    }
233
234    /// Get cache usage information
235    pub fn get_cache_info(&self) -> Result<(u64, u64)> {
236        if let Some(ref cache_dir) = self.config.cache_dir {
237            let cache_path = Path::new(cache_dir);
238            if cache_path.exists() {
239                let mut total_size = 0u64;
240                let mut file_count = 0u64;
241
242                for entry in (std::fs::read_dir(cache_path)
243                    .map_err(|e| IoError::FileError(format!("Failed to read cache dir: {}", e)))?)
244                .flatten()
245                {
246                    if let Ok(metadata) = entry.metadata() {
247                        if metadata.is_file() {
248                            total_size += metadata.len();
249                            file_count += 1;
250                        }
251                    }
252                }
253                return Ok((total_size, file_count));
254            }
255        }
256        Ok((0, 0))
257    }
258}
259
260/// Convenience functions for common network operations
261/// Download a file from URL using default client
262#[cfg(feature = "reqwest")]
263pub async fn download_file<P: AsRef<Path>>(url: &str, localpath: P) -> Result<()> {
264    let client = NetworkClient::new();
265    client.download(url, localpath).await
266}
267
268/// Upload a file to URL using default client
269#[cfg(feature = "reqwest")]
270pub async fn upload_file<P: AsRef<Path>>(localpath: P, url: &str) -> Result<()> {
271    let client = NetworkClient::new();
272    client.upload(localpath, url).await
273}
274
275/// Download a file with caching support
276#[cfg(feature = "reqwest")]
277pub async fn download_with_cache<P: AsRef<Path>>(
278    url: &str,
279    localpath: P,
280    cache_dir: Option<&str>,
281) -> Result<()> {
282    let mut client = NetworkClient::new();
283    if let Some(cache) = cache_dir {
284        client = client.with_cache_dir(cache);
285    }
286    client.download(url, localpath).await
287}
288
289/// Create a network client with cloud provider
290#[allow(dead_code)]
291pub fn create_cloud_client(provider: cloud::CloudProvider) -> NetworkClient {
292    NetworkClient::new().with_cloud_provider(provider)
293}
294
295/// Batch download multiple files
296#[cfg(feature = "reqwest")]
297pub async fn batch_download(downloads: Vec<(&str, &str)>) -> Result<Vec<Result<()>>> {
298    let client = NetworkClient::new();
299    let mut results = Vec::new();
300
301    for (url, localpath) in downloads {
302        let result = client.download(url, localpath).await;
303        results.push(result);
304    }
305
306    Ok(results)
307}
308
309/// Batch upload multiple files to cloud storage
310pub async fn batch_upload_to_cloud(
311    client: &NetworkClient,
312    uploads: Vec<(&str, &str)>,
313) -> Result<Vec<Result<()>>> {
314    let mut results = Vec::new();
315
316    for (localpath, remote_path) in uploads {
317        let result = client.upload_to_cloud(localpath, remote_path).await;
318        results.push(result);
319    }
320
321    Ok(results)
322}
323
324#[cfg(test)]
325mod tests {
326    use super::*;
327
328    #[test]
329    fn test_network_config_default() {
330        let config = NetworkConfig::default();
331        assert_eq!(config.connect_timeout, Duration::from_secs(30));
332        assert_eq!(config.read_timeout, Duration::from_secs(300));
333        assert_eq!(config.max_retries, 3);
334        assert!(config.compression);
335        assert_eq!(config.max_cache_size, 1024);
336    }
337
338    #[test]
339    fn test_network_client_creation() {
340        let client = NetworkClient::new();
341        assert!(client.cloud_provider.is_none());
342
343        let cache_dir = std::env::temp_dir().join("cache");
344        let cache_dir_str = cache_dir.to_string_lossy().into_owned();
345        let client_with_cache = NetworkClient::new().with_cache_dir(&cache_dir_str);
346        assert_eq!(client_with_cache.config.cache_dir, Some(cache_dir_str));
347    }
348
349    #[test]
350    fn test_network_config_custom() {
351        let mut headers = HashMap::new();
352        headers.insert("Authorization".to_string(), "Bearer token".to_string());
353
354        let config = NetworkConfig {
355            connect_timeout: Duration::from_secs(10),
356            read_timeout: Duration::from_secs(60),
357            max_retries: 5,
358            user_agent: "custom-agent/1.0".to_string(),
359            headers,
360            compression: false,
361            cache_dir: Some("/custom/cache".to_string()),
362            max_cache_size: 512,
363        };
364
365        let client = NetworkClient::with_config(config.clone());
366        assert_eq!(client.config.connect_timeout, Duration::from_secs(10));
367        assert_eq!(client.config.max_retries, 5);
368        assert!(!client.config.compression);
369    }
370
371    #[cfg(feature = "async")]
372    #[tokio::test]
373    async fn test_cache_operations() {
374        let temp_dir = tempfile::tempdir().expect("Operation failed");
375        let cache_path = temp_dir.path().to_str().expect("Operation failed");
376
377        let client = NetworkClient::new().with_cache_dir(cache_path);
378
379        // Test cache info on empty cache
380        let (size, count) = client.get_cache_info().expect("Operation failed");
381        assert_eq!(size, 0);
382        assert_eq!(count, 0);
383
384        // Test cache clearing
385        client.clear_cache().expect("Operation failed");
386    }
387}
scirs2_io/network/mod.rs

scirs2_io/network/
mod.rs
