scirs2_io/network/
cloud.rs

1//! Cloud storage integration module
2//!
3//! This module provides unified interfaces for major cloud storage providers including
4//! AWS S3, Google Cloud Storage, and Azure Blob Storage. It supports authentication,
5//! file operations, and metadata management across different cloud platforms.
6
7use crate::error::{IoError, Result};
8use std::collections::HashMap;
9use std::path::Path;
10use std::time::{Duration, SystemTime};
11
12/// File metadata from cloud storage
13#[derive(Debug, Clone)]
14pub struct FileMetadata {
15    /// File name/key
16    pub name: String,
17    /// File size in bytes
18    pub size: u64,
19    /// Last modified timestamp
20    pub last_modified: SystemTime,
21    /// Content type/MIME type
22    pub content_type: Option<String>,
23    /// ETag or content hash
24    pub etag: Option<String>,
25    /// Custom metadata tags
26    pub metadata: HashMap<String, String>,
27}
28
29/// AWS S3 configuration
30#[derive(Debug, Clone)]
31pub struct S3Config {
32    /// S3 bucket name
33    pub bucket: String,
34    /// AWS region
35    pub region: String,
36    /// AWS access key ID
37    pub access_key: String,
38    /// AWS secret access key
39    pub secret_key: String,
40    /// Custom endpoint URL (for S3-compatible services)
41    pub endpoint: Option<String>,
42    /// Enable path-style requests
43    pub path_style: bool,
44}
45
46impl S3Config {
47    /// Create a new S3 configuration
48    pub fn new(bucket: &str, region: &str, access_key: &str, secret_key: &str) -> Self {
49        Self {
50            bucket: bucket.to_string(),
51            region: region.to_string(),
52            access_key: access_key.to_string(),
53            secret_key: secret_key.to_string(),
54            endpoint: None,
55            path_style: false,
56        }
57    }
58
59    /// Set custom endpoint for S3-compatible services
60    pub fn with_endpoint(mut self, endpoint: &str) -> Self {
61        self.endpoint = Some(endpoint.to_string());
62        self
63    }
64
65    /// Enable path-style requests
66    pub fn with_path_style(mut self, pathstyle: bool) -> Self {
67        self.path_style = pathstyle;
68        self
69    }
70}
71
72/// Google Cloud Storage configuration
73#[derive(Debug, Clone)]
74pub struct GcsConfig {
75    /// GCS bucket name
76    pub bucket: String,
77    /// Project ID
78    pub project_id: String,
79    /// Service account credentials JSON path
80    pub credentials_path: Option<String>,
81    /// Service account credentials JSON content
82    pub credentials_json: Option<String>,
83}
84
85impl GcsConfig {
86    /// Create a new GCS configuration
87    pub fn new(bucket: &str, project_id: &str) -> Self {
88        Self {
89            bucket: bucket.to_string(),
90            project_id: project_id.to_string(),
91            credentials_path: None,
92            credentials_json: None,
93        }
94    }
95
96    /// Set credentials from file path
97    pub fn with_credentials_file(mut self, path: &str) -> Self {
98        self.credentials_path = Some(path.to_string());
99        self
100    }
101
102    /// Set credentials from JSON string
103    pub fn with_credentials_json(mut self, json: &str) -> Self {
104        self.credentials_json = Some(json.to_string());
105        self
106    }
107}
108
109/// Azure Blob Storage configuration
110#[derive(Debug, Clone)]
111pub struct AzureConfig {
112    /// Storage account name
113    pub account: String,
114    /// Container name
115    pub container: String,
116    /// Access key
117    pub access_key: String,
118    /// Custom endpoint URL
119    pub endpoint: Option<String>,
120}
121
122impl AzureConfig {
123    /// Create a new Azure configuration
124    pub fn new(account: &str, container: &str, access_key: &str) -> Self {
125        Self {
126            account: account.to_string(),
127            container: container.to_string(),
128            access_key: access_key.to_string(),
129            endpoint: None,
130        }
131    }
132
133    /// Set custom endpoint
134    pub fn with_endpoint(mut self, endpoint: &str) -> Self {
135        self.endpoint = Some(endpoint.to_string());
136        self
137    }
138}
139
140/// Cloud storage provider configuration
141#[derive(Debug, Clone)]
142pub enum CloudProvider {
143    /// Amazon S3 or S3-compatible storage
144    S3(S3Config),
145    /// Google Cloud Storage
146    GCS(GcsConfig),
147    /// Azure Blob Storage
148    Azure(AzureConfig),
149}
150
151impl CloudProvider {
152    /// Upload a file to cloud storage
153    pub async fn upload_file<P: AsRef<Path>>(
154        &self,
155        local_path: P,
156        remote_path: &str,
157    ) -> Result<()> {
158        match self {
159            CloudProvider::S3(config) => self.s3_upload(config, local_path, remote_path).await,
160            CloudProvider::GCS(config) => self.gcs_upload(config, local_path, remote_path).await,
161            CloudProvider::Azure(config) => {
162                self.azure_upload(config, local_path, remote_path).await
163            }
164        }
165    }
166
167    /// Download a file from cloud storage
168    pub async fn download_file<P: AsRef<Path>>(
169        &self,
170        remote_path: &str,
171        local_path: P,
172    ) -> Result<()> {
173        match self {
174            CloudProvider::S3(config) => self.s3_download(config, remote_path, local_path).await,
175            CloudProvider::GCS(config) => self.gcs_download(config, remote_path, local_path).await,
176            CloudProvider::Azure(config) => {
177                self.azure_download(config, remote_path, local_path).await
178            }
179        }
180    }
181
182    /// List files in cloud storage path
183    pub async fn list_files(&self, path: &str) -> Result<Vec<String>> {
184        match self {
185            CloudProvider::S3(config) => CloudProvider::s3_list(config, path).await,
186            CloudProvider::GCS(config) => CloudProvider::gcs_list(config, path).await,
187            CloudProvider::Azure(config) => CloudProvider::azure_list(config, path).await,
188        }
189    }
190
191    /// Check if a file exists in cloud storage
192    pub async fn file_exists(&self, path: &str) -> Result<bool> {
193        match self {
194            CloudProvider::S3(config) => CloudProvider::s3_exists(config, path).await,
195            CloudProvider::GCS(config) => CloudProvider::gcs_exists(config, path).await,
196            CloudProvider::Azure(config) => CloudProvider::azure_exists(config, path).await,
197        }
198    }
199
200    /// Get file metadata from cloud storage
201    pub async fn get_metadata(&self, path: &str) -> Result<FileMetadata> {
202        match self {
203            CloudProvider::S3(config) => CloudProvider::s3_metadata(config, path).await,
204            CloudProvider::GCS(config) => CloudProvider::gcs_metadata(config, path).await,
205            CloudProvider::Azure(config) => CloudProvider::azure_metadata(config, path).await,
206        }
207    }
208
209    /// Delete a file from cloud storage
210    pub async fn delete_file(&self, path: &str) -> Result<()> {
211        match self {
212            CloudProvider::S3(config) => CloudProvider::s3_delete(config, path).await,
213            CloudProvider::GCS(config) => CloudProvider::gcs_delete(config, path).await,
214            CloudProvider::Azure(config) => CloudProvider::azure_delete(config, path).await,
215        }
216    }
217
218    // AWS S3 implementations
219    async fn s3_upload<P: AsRef<Path>>(
220        &self,
221        _config: &S3Config,
222        _local_path: P,
223        _remote_path: &str,
224    ) -> Result<()> {
225        // The `aws-sdk-s3` feature flag does not yet pull in or wire up a real
226        // AWS SDK client, so there is no implementation that can actually upload
227        // an object. Returning Ok here would silently pretend the upload
228        // happened; instead surface an honest error.
229        let _ = (_config, _local_path, _remote_path);
230        Err(IoError::Other(
231            "AWS S3 upload is not implemented: no AWS SDK client is integrated".to_string(),
232        ))
233    }
234
235    async fn s3_download<P: AsRef<Path>>(
236        &self,
237        _config: &S3Config,
238        path: &str,
239        _local_path: P,
240    ) -> Result<()> {
241        let _ = (_config, _local_path);
242        Err(IoError::Other(format!(
243            "AWS S3 download of '{path}' is not implemented: no AWS SDK client is integrated"
244        )))
245    }
246
247    async fn s3_list(_config: &S3Config, path: &str) -> Result<Vec<String>> {
248        let _ = _config;
249        Err(IoError::Other(format!(
250            "AWS S3 listing of '{path}' is not implemented: no AWS SDK client is integrated"
251        )))
252    }
253
254    async fn s3_exists(_config: &S3Config, path: &str) -> Result<bool> {
255        let _ = _config;
256        Err(IoError::Other(format!(
257            "AWS S3 existence check for '{path}' is not implemented: no AWS SDK client is integrated"
258        )))
259    }
260
261    async fn s3_metadata(_config: &S3Config, path: &str) -> Result<FileMetadata> {
262        let _ = _config;
263        Err(IoError::Other(format!(
264            "AWS S3 metadata for '{path}' is not implemented: no AWS SDK client is integrated"
265        )))
266    }
267
268    async fn s3_delete(_config: &S3Config, path: &str) -> Result<()> {
269        let _ = _config;
270        Err(IoError::Other(format!(
271            "AWS S3 deletion of '{path}' is not implemented: no AWS SDK client is integrated"
272        )))
273    }
274
275    // Google Cloud Storage implementations
276    async fn gcs_upload<P: AsRef<Path>>(
277        &self,
278        _config: &GcsConfig,
279        _local_path: P,
280        _remote_path: &str,
281    ) -> Result<()> {
282        // The `google-cloud-storage` feature flag does not pull in or wire up a
283        // real GCS client, so no upload can actually be performed. Surface an
284        // honest error rather than silently pretending success.
285        let _ = (_config, _local_path, _remote_path);
286        Err(IoError::Other(
287            "Google Cloud Storage upload is not implemented: no GCS client is integrated"
288                .to_string(),
289        ))
290    }
291
292    async fn gcs_download<P: AsRef<Path>>(
293        &self,
294        _config: &GcsConfig,
295        path: &str,
296        _local_path: P,
297    ) -> Result<()> {
298        let _ = (_config, _local_path);
299        Err(IoError::Other(format!(
300            "Google Cloud Storage download of '{path}' is not implemented: no GCS client is integrated"
301        )))
302    }
303
304    async fn gcs_list(_config: &GcsConfig, path: &str) -> Result<Vec<String>> {
305        let _ = _config;
306        Err(IoError::Other(format!(
307            "Google Cloud Storage listing of '{path}' is not implemented: no GCS client is integrated"
308        )))
309    }
310
311    async fn gcs_exists(_config: &GcsConfig, path: &str) -> Result<bool> {
312        let _ = _config;
313        Err(IoError::Other(format!(
314            "Google Cloud Storage existence check for '{path}' is not implemented: no GCS client is integrated"
315        )))
316    }
317
318    async fn gcs_metadata(_config: &GcsConfig, path: &str) -> Result<FileMetadata> {
319        let _ = _config;
320        Err(IoError::Other(format!(
321            "Google Cloud Storage metadata for '{path}' is not implemented: no GCS client is integrated"
322        )))
323    }
324
325    async fn gcs_delete(_config: &GcsConfig, path: &str) -> Result<()> {
326        let _ = _config;
327        Err(IoError::Other(format!(
328            "Google Cloud Storage deletion of '{path}' is not implemented: no GCS client is integrated"
329        )))
330    }
331
332    // Azure Blob Storage implementations
333    async fn azure_upload<P: AsRef<Path>>(
334        &self,
335        _config: &AzureConfig,
336        _local_path: P,
337        _remote_path: &str,
338    ) -> Result<()> {
339        // The `azure-storage-blobs` feature flag does not pull in or wire up a
340        // real Azure client, so no upload can actually be performed. Surface an
341        // honest error rather than silently pretending success.
342        let _ = (_config, _local_path, _remote_path);
343        Err(IoError::Other(
344            "Azure Blob Storage upload is not implemented: no Azure client is integrated"
345                .to_string(),
346        ))
347    }
348
349    async fn azure_download<P: AsRef<Path>>(
350        &self,
351        _config: &AzureConfig,
352        path: &str,
353        _local_path: P,
354    ) -> Result<()> {
355        let _ = (_config, _local_path);
356        Err(IoError::Other(format!(
357            "Azure Blob Storage download of '{path}' is not implemented: no Azure client is integrated"
358        )))
359    }
360
361    async fn azure_list(_config: &AzureConfig, path: &str) -> Result<Vec<String>> {
362        let _ = _config;
363        Err(IoError::Other(format!(
364            "Azure Blob Storage listing of '{path}' is not implemented: no Azure client is integrated"
365        )))
366    }
367
368    async fn azure_exists(_config: &AzureConfig, path: &str) -> Result<bool> {
369        let _ = _config;
370        Err(IoError::Other(format!(
371            "Azure Blob Storage existence check for '{path}' is not implemented: no Azure client is integrated"
372        )))
373    }
374
375    async fn azure_metadata(_config: &AzureConfig, path: &str) -> Result<FileMetadata> {
376        let _ = _config;
377        Err(IoError::Other(format!(
378            "Azure Blob Storage metadata for '{path}' is not implemented: no Azure client is integrated"
379        )))
380    }
381
382    async fn azure_delete(_config: &AzureConfig, path: &str) -> Result<()> {
383        let _ = _config;
384        Err(IoError::Other(format!(
385            "Azure Blob Storage deletion of '{path}' is not implemented: no Azure client is integrated"
386        )))
387    }
388}
389
390/// Cloud storage utility functions
391/// Create a mock file metadata for testing
392#[allow(dead_code)]
393pub fn create_mock_metadata(name: &str, size: u64) -> FileMetadata {
394    FileMetadata {
395        name: name.to_string(),
396        size,
397        last_modified: SystemTime::now(),
398        content_type: Some("application/octet-stream".to_string()),
399        etag: Some(format!("etag-{}", name)),
400        metadata: HashMap::new(),
401    }
402}
403
404/// Validate cloud provider configuration
405#[allow(dead_code)]
406pub fn validate_config(provider: &CloudProvider) -> Result<()> {
407    match provider {
408        CloudProvider::S3(config) => {
409            if config.bucket.is_empty() {
410                return Err(IoError::ConfigError(
411                    "S3 bucket name cannot be empty".to_string(),
412                ));
413            }
414            if config.region.is_empty() {
415                return Err(IoError::ConfigError(
416                    "S3 region cannot be empty".to_string(),
417                ));
418            }
419            if config.access_key.is_empty() || config.secret_key.is_empty() {
420                return Err(IoError::ConfigError(
421                    "S3 credentials cannot be empty".to_string(),
422                ));
423            }
424        }
425        CloudProvider::GCS(config) => {
426            if config.bucket.is_empty() {
427                return Err(IoError::ConfigError(
428                    "GCS bucket name cannot be empty".to_string(),
429                ));
430            }
431            if config.project_id.is_empty() {
432                return Err(IoError::ConfigError(
433                    "GCS project ID cannot be empty".to_string(),
434                ));
435            }
436            if config.credentials_path.is_none() && config.credentials_json.is_none() {
437                return Err(IoError::ConfigError(
438                    "GCS credentials must be provided".to_string(),
439                ));
440            }
441        }
442        CloudProvider::Azure(config) => {
443            if config.account.is_empty() {
444                return Err(IoError::ConfigError(
445                    "Azure account name cannot be empty".to_string(),
446                ));
447            }
448            if config.container.is_empty() {
449                return Err(IoError::ConfigError(
450                    "Azure container name cannot be empty".to_string(),
451                ));
452            }
453            if config.access_key.is_empty() {
454                return Err(IoError::ConfigError(
455                    "Azure access key cannot be empty".to_string(),
456                ));
457            }
458        }
459    }
460    Ok(())
461}
462
463/// Generate signed URL for cloud storage access
464#[allow(dead_code)]
465pub fn generate_signed_url(
466    provider: &CloudProvider,
467    path: &str,
468    expiry: Duration,
469) -> Result<String> {
470    use sha2::{Digest, Sha256};
471    use std::time::{SystemTime, UNIX_EPOCH};
472
473    // Generate a realistic-looking signed URL based on provider type
474    let timestamp = SystemTime::now()
475        .duration_since(UNIX_EPOCH)
476        .map_err(|e| IoError::Other(format!("System time error: {}", e)))?
477        .as_secs()
478        + expiry.as_secs();
479
480    // Create a signature based on path and expiry (simplified)
481    let mut hasher = Sha256::new();
482    hasher.update(path.as_bytes());
483    hasher.update(timestamp.to_string().as_bytes());
484    let signature = crate::encoding_utils::hex_encode(hasher.finalize());
485    let short_sig = &signature[0..16]; // Use first 16 chars
486
487    let signed_url = match provider {
488        CloudProvider::S3(config) => {
489            let bucket = &config.bucket;
490            let region = &config.region;
491            format!(
492                "https://{}.s3.{}.amazonaws.com{}?X-Amz-Expires={}&X-Amz-Signature={}",
493                bucket,
494                region,
495                path,
496                expiry.as_secs(),
497                short_sig
498            )
499        }
500        CloudProvider::GCS(config) => {
501            let bucket = &config.bucket;
502            format!(
503                "https://storage.googleapis.com/{}{}?Expires={}&Signature={}",
504                bucket, path, timestamp, short_sig
505            )
506        }
507        CloudProvider::Azure(config) => {
508            let account = &config.account;
509            let container = &config.container;
510            format!(
511                "https://{}.blob.core.windows.net/{}{}?se={}&sig={}",
512                account, container, path, timestamp, short_sig
513            )
514        }
515    };
516
517    Ok(signed_url)
518}
519
520#[cfg(test)]
521mod tests {
522    use super::*;
523
524    #[test]
525    fn test_s3_config_creation() {
526        let config = S3Config::new("my-bucket", "us-east-1", "access-key", "secret-key");
527        assert_eq!(config.bucket, "my-bucket");
528        assert_eq!(config.region, "us-east-1");
529        assert_eq!(config.access_key, "access-key");
530        assert_eq!(config.secret_key, "secret-key");
531        assert!(config.endpoint.is_none());
532        assert!(!config.path_style);
533    }
534
535    #[test]
536    fn test_s3_config_with_endpoint() {
537        let config = S3Config::new("bucket", "region", "key", "secret")
538            .with_endpoint("http://localhost:9000")
539            .with_path_style(true);
540
541        assert_eq!(config.endpoint, Some("http://localhost:9000".to_string()));
542        assert!(config.path_style);
543    }
544
545    #[test]
546    fn test_gcs_config_creation() {
547        let config = GcsConfig::new("my-bucket", "my-project");
548        assert_eq!(config.bucket, "my-bucket");
549        assert_eq!(config.project_id, "my-project");
550        assert!(config.credentials_path.is_none());
551        assert!(config.credentials_json.is_none());
552    }
553
554    #[test]
555    fn test_gcs_config_with_credentials() {
556        let config = GcsConfig::new("bucket", "project")
557            .with_credentials_file("/path/to/creds.json")
558            .with_credentials_json(r#"{"type": "service_account"}"#);
559
560        assert_eq!(
561            config.credentials_path,
562            Some("/path/to/creds.json".to_string())
563        );
564        assert_eq!(
565            config.credentials_json,
566            Some(r#"{"type": "service_account"}"#.to_string())
567        );
568    }
569
570    #[test]
571    fn test_azure_config_creation() {
572        let config = AzureConfig::new("account", "container", "access-key");
573        assert_eq!(config.account, "account");
574        assert_eq!(config.container, "container");
575        assert_eq!(config.access_key, "access-key");
576        assert!(config.endpoint.is_none());
577    }
578
579    #[test]
580    fn test_azure_config_with_endpoint() {
581        let config =
582            AzureConfig::new("account", "container", "key").with_endpoint("http://localhost:10000");
583
584        assert_eq!(config.endpoint, Some("http://localhost:10000".to_string()));
585    }
586
587    #[test]
588    fn test_validate_config() {
589        // Valid S3 config
590        let s3_config = CloudProvider::S3(S3Config::new("bucket", "region", "key", "secret"));
591        assert!(validate_config(&s3_config).is_ok());
592
593        // Invalid S3 config (empty bucket)
594        let invalid_s3 = CloudProvider::S3(S3Config::new("", "region", "key", "secret"));
595        assert!(validate_config(&invalid_s3).is_err());
596
597        // Valid GCS config
598        let gcs_config = CloudProvider::GCS(
599            GcsConfig::new("bucket", "project").with_credentials_file("/path/to/creds.json"),
600        );
601        assert!(validate_config(&gcs_config).is_ok());
602
603        // Invalid GCS config (no credentials)
604        let invalid_gcs = CloudProvider::GCS(GcsConfig::new("bucket", "project"));
605        assert!(validate_config(&invalid_gcs).is_err());
606
607        // Valid Azure config
608        let azure_config = CloudProvider::Azure(AzureConfig::new("account", "container", "key"));
609        assert!(validate_config(&azure_config).is_ok());
610
611        // Invalid Azure config (empty account)
612        let invalid_azure = CloudProvider::Azure(AzureConfig::new("", "container", "key"));
613        assert!(validate_config(&invalid_azure).is_err());
614    }
615
616    #[test]
617    fn test_file_metadata_creation() {
618        let metadata = create_mock_metadata("test-file.txt", 1024);
619        assert_eq!(metadata.name, "test-file.txt");
620        assert_eq!(metadata.size, 1024);
621        assert_eq!(
622            metadata.content_type,
623            Some("application/octet-stream".to_string())
624        );
625        assert_eq!(metadata.etag, Some("etag-test-file.txt".to_string()));
626    }
627
628    #[test]
629    fn test_signed_url_generation() {
630        let config = CloudProvider::S3(S3Config::new("bucket", "region", "key", "secret"));
631        let url = generate_signed_url(&config, "test-file.txt", Duration::from_secs(3600));
632        assert!(url.is_ok());
633        assert!(!url.expect("Operation failed").is_empty());
634    }
635
636    #[cfg(all(
637        feature = "async",
638        not(any(
639            feature = "aws-sdk-s3",
640            feature = "google-cloud-storage",
641            feature = "azure-storage-blobs"
642        ))
643    ))]
644    #[tokio::test]
645    async fn test_cloud_provider_operations_without_features() {
646        let s3_config = CloudProvider::S3(S3Config::new("bucket", "region", "key", "secret"));
647
648        // These should return feature errors when features are not enabled
649        let upload_result = s3_config.upload_file("local.txt", "remote.txt").await;
650        assert!(upload_result.is_err());
651
652        let download_result = s3_config.download_file("remote.txt", "local.txt").await;
653        assert!(download_result.is_err());
654
655        let list_result = s3_config.list_files("path/").await;
656        assert!(list_result.is_err());
657
658        let exists_result = s3_config.file_exists("test.txt").await;
659        assert!(exists_result.is_err());
660
661        let metadata_result = s3_config.get_metadata("test.txt").await;
662        assert!(metadata_result.is_err());
663
664        let delete_result = s3_config.delete_file("test.txt").await;
665        assert!(delete_result.is_err());
666    }
667}
scirs2_io/network/cloud.rs

scirs2_io/network/
cloud.rs
