backbone_storage/
lib.rs

1//! Backbone Storage Module
2//!
3//! Provides file storage capabilities with multiple backend support:
4//! - Local filesystem storage
5//! - AWS S3 / S3-compatible (MinIO) storage (feature: `s3`)
6//! - File compression and optimization (feature: `compression`)
7//! - Security and virus scanning (feature: `security`)
8
9pub mod error;
10pub mod types;
11pub mod traits;
12
13// Use simple local storage for stability
14#[path = "local_simple.rs"]
15pub mod local;
16
17// S3 / MinIO (feature-gated)
18#[cfg(feature = "s3")]
19pub mod s3;
20#[cfg(feature = "s3")]
21pub mod minio;
22
23// Conditionally compile advanced features
24#[cfg(feature = "compression")]
25pub mod compression;
26
27#[cfg(feature = "security")]
28pub mod security;
29
30// Re-export main types for convenient use
31pub use error::{StorageError, StorageResult};
32pub use types::{
33    StorageFile, StorageConfig, StorageBackend, UploadOptions,
34    DownloadOptions, PresignedUrlOptions, FileMetadata, StorageStats,
35    HttpMethod, ByteRange, LocalStorageConfig,
36    S3StorageConfig, MinIOStorageConfig, GcsStorageConfig,
37};
38pub use traits::{StorageService, StorageManager, StorageMonitor, FileListResult};
39
40#[cfg(feature = "compression")]
41pub use compression::{
42    ImageCompressor, CompressionResult, CompressionQuality,
43    FileCategory, CompressionAlgorithm, CompressionConfig,
44    TextCompressionConfig, DocumentCompressionConfig,
45};
46
47#[cfg(feature = "security")]
48pub use security::{
49    SecurityEngine, SecurityConfig, ThreatLevel, SecurityAnalysis,
50    ExecutableMetadata, DigitalSignature,
51    Threat, ThreatType, SizeAnalysis, ContentAnalysis,
52};
53
54
55// Default configurations
56pub const DEFAULT_MULTIPART_THRESHOLD: u64 = 100 * 1024 * 1024; // 100MB
57pub const DEFAULT_CHUNK_SIZE: usize = 8 * 1024 * 1024; // 8MB
58pub const DEFAULT_PRESIGNED_EXPIRY: u64 = 3600; // 1 hour
59
60/// Storage module version
61pub const VERSION: &str = env!("CARGO_PKG_VERSION");
62
63/// Get default storage configuration
64pub fn default_config() -> StorageConfig {
65    StorageConfig::default()
66}
67
68/// Require an environment variable to be set and non-empty.
69#[cfg(any(feature = "s3", test))]
70fn require_env_non_empty(var_name: &str) -> StorageResult<String> {
71    let val = std::env::var(var_name)
72        .map_err(|_| StorageError::InvalidConfiguration(format!("{var_name} is required but not set")))?;
73    if val.trim().is_empty() {
74        return Err(StorageError::InvalidConfiguration(format!("{var_name} is required but empty")));
75    }
76    Ok(val)
77}
78
79/// Log a warning if a credential env var is empty (empty may be valid for IAM roles).
80#[cfg(feature = "s3")]
81fn warn_if_empty(var_name: &str, value: &str) {
82    if value.is_empty() {
83        tracing::warn!(var = var_name, "Credential is empty; falling back to SDK default chain");
84    }
85}
86
87/// Create storage service from environment variables
88pub async fn from_env() -> StorageResult<Box<dyn StorageService>> {
89    let backend = std::env::var("STORAGE_BACKEND")
90        .unwrap_or_else(|_| "local".to_string())
91        .parse()
92        .map_err(|_| StorageError::InvalidConfiguration("STORAGE_BACKEND".to_string()))?;
93
94    let config = StorageConfig::from_env()?;
95
96    match backend {
97        StorageBackend::Local => {
98            let local_config = config.into_local_config()
99                .ok_or_else(|| StorageError::InvalidConfiguration("Local storage configuration missing".to_string()))?;
100            Ok(Box::new(crate::local::LocalStorage::new(local_config)?))
101        }
102        #[cfg(feature = "s3")]
103        StorageBackend::S3 => {
104            let bucket = std::env::var("S3_BUCKET").unwrap_or_else(|_| config.bucket.clone());
105            if bucket.is_empty() || bucket == "default" {
106                return Err(StorageError::InvalidConfiguration(
107                    "S3_BUCKET (or STORAGE_BUCKET) is required for S3 backend".to_string(),
108                ));
109            }
110            let access_key_id = std::env::var("AWS_ACCESS_KEY_ID").unwrap_or_default();
111            let secret_access_key = std::env::var("AWS_SECRET_ACCESS_KEY").unwrap_or_default();
112            warn_if_empty("AWS_ACCESS_KEY_ID", &access_key_id);
113            warn_if_empty("AWS_SECRET_ACCESS_KEY", &secret_access_key);
114
115            let s3_config = S3StorageConfig {
116                region: std::env::var("AWS_REGION").unwrap_or_else(|_| "us-east-1".to_string()),
117                access_key_id,
118                secret_access_key,
119                bucket,
120                endpoint: std::env::var("S3_ENDPOINT").ok(),
121                base_path: config.base_path.clone(),
122                path_style: std::env::var("S3_PATH_STYLE").map(|v| v == "true").unwrap_or(false),
123                ..Default::default()
124            };
125            Ok(Box::new(crate::s3::S3Storage::new(s3_config).await?))
126        }
127        #[cfg(not(feature = "s3"))]
128        StorageBackend::S3 => {
129            Err(StorageError::UnsupportedOperation {
130                operation: "storage creation".to_string(),
131                backend: "s3 (enable the 's3' feature)".to_string(),
132            })
133        }
134        #[cfg(feature = "s3")]
135        StorageBackend::MinIO => {
136            let endpoint = require_env_non_empty("MINIO_ENDPOINT")?;
137            let access_key = std::env::var("MINIO_ACCESS_KEY")
138                .or_else(|_| std::env::var("AWS_ACCESS_KEY_ID"))
139                .unwrap_or_default();
140            let secret_key = std::env::var("MINIO_SECRET_KEY")
141                .or_else(|_| std::env::var("AWS_SECRET_ACCESS_KEY"))
142                .unwrap_or_default();
143            if access_key.is_empty() {
144                return Err(StorageError::InvalidConfiguration(
145                    "MINIO_ACCESS_KEY (or AWS_ACCESS_KEY_ID) is required for MinIO backend".to_string(),
146                ));
147            }
148            if secret_key.is_empty() {
149                return Err(StorageError::InvalidConfiguration(
150                    "MINIO_SECRET_KEY (or AWS_SECRET_ACCESS_KEY) is required for MinIO backend".to_string(),
151                ));
152            }
153            let bucket = std::env::var("MINIO_BUCKET").unwrap_or_else(|_| config.bucket.clone());
154            if bucket.is_empty() || bucket == "default" {
155                return Err(StorageError::InvalidConfiguration(
156                    "MINIO_BUCKET (or STORAGE_BUCKET) is required for MinIO backend".to_string(),
157                ));
158            }
159
160            let minio_config = MinIOStorageConfig {
161                endpoint,
162                access_key,
163                secret_key,
164                bucket,
165                region: std::env::var("MINIO_REGION").unwrap_or_else(|_| "us-east-1".to_string()),
166                base_path: config.base_path.clone(),
167                ..Default::default()
168            };
169            Ok(Box::new(crate::minio::MinIOStorage::new(minio_config).await?))
170        }
171        #[cfg(not(feature = "s3"))]
172        StorageBackend::MinIO => {
173            Err(StorageError::UnsupportedOperation {
174                operation: "storage creation".to_string(),
175                backend: "minio (enable the 's3' feature)".to_string(),
176            })
177        }
178        StorageBackend::AzureBlob => {
179            Err(StorageError::UnsupportedOperation {
180                operation: "storage creation".to_string(),
181                backend: "azure_blob".to_string(),
182            })
183        }
184        StorageBackend::GCS => {
185            Err(StorageError::UnsupportedOperation {
186                operation: "storage creation".to_string(),
187                backend: "gcs (not yet implemented)".to_string(),
188            })
189        }
190    }
191}
192
193/// Utility function to detect file MIME type
194pub fn detect_mime_type(file_path: &str) -> String {
195    mime_guess::from_path(file_path)
196        .first_or_octet_stream()
197        .to_string()
198}
199
200/// Utility function to generate safe file names
201pub fn sanitize_filename(filename: &str) -> String {
202    use std::path::Component;
203
204    let path = std::path::Path::new(filename);
205    let mut result = String::new();
206
207    for component in path.components() {
208        match component {
209            Component::Normal(s) => {
210                if let Some(s) = s.to_str() {
211                    // Replace potentially problematic characters
212                    let sanitized = s
213                        .chars()
214                        .map(|c| match c {
215                            '/' | '\\' | ':' | '*' | '?' | '"' | '<' | '>' | '|' => '_',
216                            c if c.is_control() => '_',
217                            c => c,
218                        })
219                        .collect::<String>();
220
221                    if !result.is_empty() {
222                        result.push('/');
223                    }
224                    result.push_str(&sanitized);
225                }
226            }
227            Component::RootDir => continue,
228            Component::CurDir => continue,
229            Component::ParentDir => {
230                if !result.is_empty() {
231                    result.push('/');
232                }
233                result.push_str("_parent");
234            }
235            Component::Prefix(_) => continue,
236        }
237    }
238
239    // Ensure the result is not empty
240    if result.is_empty() {
241        result.push_str("file");
242    }
243
244    result
245}
246
247#[cfg(test)]
248mod tests {
249    use super::*;
250
251    #[test]
252    fn test_mime_type_detection() {
253        assert_eq!(detect_mime_type("test.txt"), "text/plain");
254        assert_eq!(detect_mime_type("test.json"), "application/json");
255        assert_eq!(detect_mime_type("test.jpg"), "image/jpeg");
256        assert_eq!(detect_mime_type("unknown.totally_unknown_ext"), "application/octet-stream");
257    }
258
259    #[test]
260    fn test_filename_sanitization() {
261        assert_eq!(sanitize_filename("normal_file.txt"), "normal_file.txt");
262        assert_eq!(sanitize_filename("file/with\\slashes"), "file/with_slashes");
263        assert_eq!(sanitize_filename("../parent/directory"), "_parent/parent/directory");
264        assert_eq!(sanitize_filename("file:with*special?chars"), "file_with_special_chars");
265        assert_eq!(sanitize_filename(""), "file");
266    }
267
268    #[test]
269    fn test_default_config() {
270        let config = default_config();
271        assert_eq!(config.backend, StorageBackend::Local);
272        assert_eq!(config.multipart_threshold, DEFAULT_MULTIPART_THRESHOLD);
273    }
274
275    #[test]
276    fn test_version() {
277        assert!(!VERSION.is_empty());
278    }
279
280    #[test]
281    fn test_require_env_non_empty_missing() {
282        // Use a unique env var name that definitely doesn't exist
283        let result = require_env_non_empty("__BACKBONE_TEST_NONEXISTENT_VAR__");
284        assert!(result.is_err());
285        let msg = result.unwrap_err().to_string();
286        assert!(msg.contains("required but not set"), "got: {msg}");
287    }
288
289    #[test]
290    fn test_require_env_non_empty_blank() {
291        let key = "__BACKBONE_TEST_EMPTY_VAR__";
292        std::env::set_var(key, "   ");
293        let result = require_env_non_empty(key);
294        std::env::remove_var(key);
295        assert!(result.is_err());
296        let msg = result.unwrap_err().to_string();
297        assert!(msg.contains("required but empty"), "got: {msg}");
298    }
299
300    #[test]
301    fn test_require_env_non_empty_ok() {
302        let key = "__BACKBONE_TEST_OK_VAR__";
303        std::env::set_var(key, "value");
304        let result = require_env_non_empty(key);
305        std::env::remove_var(key);
306        assert_eq!(result.unwrap(), "value");
307    }
308}
backbone_storage/lib.rs

backbone_storage/
lib.rs
