backbone_storage/
types.rs

1//! Storage module types and structures
2
3use serde::{Serialize, Deserialize};
4use std::collections::HashMap;
5use chrono::{DateTime, Utc};
6use uuid::Uuid;
7
8/// Storage backend types
9#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
10pub enum StorageBackend {
11    /// Local filesystem storage
12    #[default]
13    Local,
14    /// Amazon S3
15    S3,
16    /// MinIO (S3-compatible)
17    MinIO,
18    /// Azure Blob Storage (future)
19    AzureBlob,
20    /// Google Cloud Storage
21    GCS,
22}
23
24impl std::fmt::Display for StorageBackend {
25    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
26        match self {
27            Self::Local => write!(f, "local"),
28            Self::S3 => write!(f, "s3"),
29            Self::MinIO => write!(f, "minio"),
30            Self::AzureBlob => write!(f, "azure_blob"),
31            Self::GCS => write!(f, "gcs"),
32        }
33    }
34}
35
36impl std::str::FromStr for StorageBackend {
37    type Err = crate::error::StorageError;
38
39    fn from_str(s: &str) -> Result<Self, Self::Err> {
40        match s.to_lowercase().as_str() {
41            "local" => Ok(Self::Local),
42            "s3" | "aws" => Ok(Self::S3),
43            "minio" => Ok(Self::MinIO),
44            "azure_blob" | "azure" => Ok(Self::AzureBlob),
45            "gcs" | "google" => Ok(Self::GCS),
46            _ => Err(crate::error::StorageError::InvalidConfiguration(
47                format!("Invalid storage backend: {}", s)
48            )),
49        }
50    }
51}
52
53/// Storage configuration
54#[derive(Debug, Clone, Serialize, Deserialize)]
55pub struct StorageConfig {
56    /// Storage backend type
57    pub backend: StorageBackend,
58
59    /// Default bucket/container name
60    pub bucket: String,
61
62    /// Base path/prefix for all files
63    pub base_path: Option<String>,
64
65    /// Multipart upload threshold in bytes
66    pub multipart_threshold: u64,
67
68    /// Chunk size for multipart uploads in bytes
69    pub chunk_size: usize,
70
71    /// Enable file versioning
72    pub versioning: bool,
73
74    /// Enable compression
75    pub compression: bool,
76
77    /// Encryption settings
78    pub encryption: Option<EncryptionConfig>,
79
80    /// Comprehensive compression settings (requires 'compression' feature)
81    #[cfg(feature = "compression")]
82    pub compression_config: Option<crate::CompressionConfig>,
83
84    /// Retention policy
85    pub retention: Option<RetentionConfig>,
86
87    /// Access control settings
88    pub access_control: Option<AccessControlConfig>,
89
90    /// Backend-specific configurations
91    pub backend_config: HashMap<String, serde_json::Value>,
92}
93
94impl Default for StorageConfig {
95    fn default() -> Self {
96        Self {
97            backend: StorageBackend::Local,
98            bucket: "default".to_string(),
99            base_path: None,
100            multipart_threshold: crate::DEFAULT_MULTIPART_THRESHOLD,
101            chunk_size: crate::DEFAULT_CHUNK_SIZE,
102            versioning: false,
103            compression: false,
104            encryption: None,
105            #[cfg(feature = "compression")]
106            compression_config: None,
107            retention: None,
108            access_control: None,
109            backend_config: HashMap::new(),
110        }
111    }
112}
113
114impl StorageConfig {
115    /// Create configuration from environment variables
116    pub fn from_env() -> crate::StorageResult<Self> {
117        let backend_str = std::env::var("STORAGE_BACKEND")
118            .unwrap_or_else(|_| "local".to_string());
119        let backend = backend_str.parse()?;
120
121        let mut config = Self {
122            backend,
123            bucket: std::env::var("STORAGE_BUCKET")
124                .unwrap_or_else(|_| "default".to_string()),
125            base_path: std::env::var("STORAGE_BASE_PATH").ok(),
126            ..Default::default()
127        };
128
129        // Load backend-specific configurations
130        if config.backend == StorageBackend::Local {
131            config.backend_config.insert("base_dir".to_string(),
132                serde_json::Value::String(
133                    std::env::var("LOCAL_STORAGE_DIR").unwrap_or_else(|_| "./storage".to_string())
134                )
135            );
136        }
137
138        Ok(config)
139    }
140
141    
142    /// Convert to local storage configuration
143    pub fn into_local_config(self) -> Option<LocalStorageConfig> {
144        if self.backend != StorageBackend::Local {
145            return None;
146        }
147
148        Some(LocalStorageConfig {
149            base_dir: self.backend_config.get("base_dir")
150                .and_then(|v| v.as_str())
151                .map(|s| s.to_string())
152                .unwrap_or_else(|| "./storage".to_string()),
153            base_path: self.base_path,
154            compression: self.compression,
155            encryption: self.encryption,
156        })
157    }
158}
159
160
161/// Local storage configuration
162#[derive(Debug, Clone, Serialize, Deserialize)]
163pub struct LocalStorageConfig {
164    pub base_dir: String,
165    pub base_path: Option<String>,
166    pub compression: bool,
167    pub encryption: Option<EncryptionConfig>,
168}
169
170/// AWS S3 storage configuration
171#[derive(Debug, Clone, Serialize, Deserialize)]
172pub struct S3StorageConfig {
173    /// AWS region (e.g., "us-east-1")
174    pub region: String,
175    /// AWS access key ID
176    pub access_key_id: String,
177    /// AWS secret access key
178    pub secret_access_key: String,
179    /// S3 bucket name
180    pub bucket: String,
181    /// Custom endpoint URL (for S3-compatible services)
182    pub endpoint: Option<String>,
183    /// Base path prefix for all objects
184    pub base_path: Option<String>,
185    /// Multipart upload threshold in bytes
186    pub multipart_threshold: u64,
187    /// Chunk size for multipart uploads
188    pub chunk_size: usize,
189    /// Encryption settings
190    pub encryption: Option<EncryptionConfig>,
191    /// Use path-style URLs (required for some S3-compatible services)
192    pub path_style: bool,
193    /// Enable versioning
194    pub versioning: bool,
195}
196
197impl Default for S3StorageConfig {
198    fn default() -> Self {
199        Self {
200            region: "us-east-1".to_string(),
201            access_key_id: String::new(),
202            secret_access_key: String::new(),
203            bucket: String::new(),
204            endpoint: None,
205            base_path: None,
206            multipart_threshold: crate::DEFAULT_MULTIPART_THRESHOLD,
207            chunk_size: crate::DEFAULT_CHUNK_SIZE,
208            encryption: None,
209            path_style: false,
210            versioning: false,
211        }
212    }
213}
214
215/// MinIO storage configuration (S3-compatible)
216#[derive(Debug, Clone, Serialize, Deserialize)]
217pub struct MinIOStorageConfig {
218    /// MinIO endpoint URL (e.g., "http://localhost:9000")
219    pub endpoint: String,
220    /// MinIO access key
221    pub access_key: String,
222    /// MinIO secret key
223    pub secret_key: String,
224    /// Bucket name
225    pub bucket: String,
226    /// Region (usually "us-east-1" for MinIO)
227    pub region: String,
228    /// Base path prefix for all objects
229    pub base_path: Option<String>,
230    /// Use SSL/TLS
231    pub use_ssl: bool,
232    /// Multipart upload threshold in bytes
233    pub multipart_threshold: u64,
234    /// Chunk size for multipart uploads
235    pub chunk_size: usize,
236    /// Encryption settings
237    pub encryption: Option<EncryptionConfig>,
238    /// Enable versioning
239    pub versioning: bool,
240}
241
242impl Default for MinIOStorageConfig {
243    fn default() -> Self {
244        Self {
245            endpoint: "http://localhost:9000".to_string(),
246            access_key: String::new(),
247            secret_key: String::new(),
248            bucket: String::new(),
249            region: "us-east-1".to_string(),
250            base_path: None,
251            use_ssl: false,
252            multipart_threshold: crate::DEFAULT_MULTIPART_THRESHOLD,
253            chunk_size: crate::DEFAULT_CHUNK_SIZE,
254            encryption: None,
255            versioning: false,
256        }
257    }
258}
259
260/// Google Cloud Storage configuration
261#[derive(Debug, Clone, Serialize, Deserialize)]
262pub struct GcsStorageConfig {
263    /// GCP project ID
264    pub project_id: String,
265    /// GCS bucket name
266    pub bucket: String,
267    /// Path to service account credentials JSON file
268    pub credentials_json: Option<String>,
269    /// Base path prefix for all objects
270    pub base_path: Option<String>,
271}
272
273impl Default for GcsStorageConfig {
274    fn default() -> Self {
275        Self {
276            project_id: String::new(),
277            bucket: String::new(),
278            credentials_json: None,
279            base_path: None,
280        }
281    }
282}
283
284/// Encryption configuration
285#[derive(Debug, Clone, Serialize, Deserialize)]
286pub struct EncryptionConfig {
287    /// Encryption algorithm
288    pub algorithm: EncryptionAlgorithm,
289
290    /// Encryption key (base64 encoded)
291    pub key: Option<String>,
292
293    /// Use server-side encryption for cloud storage
294    pub server_side: bool,
295
296    /// Additional encryption parameters
297    pub parameters: HashMap<String, String>,
298}
299
300/// Image compression configuration
301#[derive(Debug, Clone, Serialize, Deserialize)]
302pub struct ImageCompressionConfig {
303    /// Enable automatic image compression
304    pub enabled: bool,
305
306    /// Compression quality level
307    pub quality: u8, // 0-100
308
309    /// Maximum width in pixels (resizes larger images)
310    pub max_width: Option<u32>,
311
312    /// Maximum height in pixels (resizes larger images)
313    pub max_height: Option<u32>,
314
315    /// Maximum file size in bytes (recompresses if larger)
316    pub max_file_size: Option<u64>,
317
318    /// Target formats for conversion (e.g., "webp", "jpeg", "png")
319    pub preferred_formats: Vec<String>,
320
321    /// Preserve metadata (EXIF, etc.)
322    pub preserve_metadata: bool,
323
324    /// Progressive JPEG encoding
325    pub progressive_jpeg: bool,
326
327    /// WebP compression method (0-6, higher = slower but better compression)
328    pub webp_method: u8,
329}
330
331impl Default for ImageCompressionConfig {
332    fn default() -> Self {
333        Self {
334            enabled: true,
335            quality: 80, // Medium quality
336            max_width: Some(2048),
337            max_height: Some(2048),
338            max_file_size: Some(2 * 1024 * 1024), // 2MB
339            preferred_formats: vec!["webp".to_string(), "jpeg".to_string()],
340            preserve_metadata: false,
341            progressive_jpeg: true,
342            webp_method: 4,
343        }
344    }
345}
346
347/// Encryption algorithms
348#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
349pub enum EncryptionAlgorithm {
350    /// AES-256-GCM
351    AES256GCM,
352    /// AES-256-CBC
353    AES256CBC,
354    /// ChaCha20-Poly1305
355    ChaCha20Poly1305,
356}
357
358/// Retention policy configuration
359#[derive(Debug, Clone, Serialize, Deserialize)]
360pub struct RetentionConfig {
361    /// Retention mode
362    pub mode: RetentionMode,
363
364    /// Retention period in days
365    pub period_days: u32,
366
367    /// Apply to all files or specific patterns
368    pub patterns: Vec<String>,
369}
370
371/// Retention modes
372#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
373pub enum RetentionMode {
374    /// Keep files for N days
375    KeepForDays,
376    /// Delete files older than N days
377    DeleteAfterDays,
378    /// Move to glacier after N days
379    GlacierAfterDays,
380}
381
382/// Access control configuration
383#[derive(Debug, Clone, Serialize, Deserialize)]
384pub struct AccessControlConfig {
385    /// Default access policy
386    pub default_policy: AccessPolicy,
387
388    /// CORS configuration
389    pub cors: Option<CorsConfig>,
390}
391
392/// Access policies
393#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
394pub enum AccessPolicy {
395    /// Private access
396    Private,
397    /// Public read
398    PublicRead,
399    /// Public read/write
400    PublicReadWrite,
401    /// Custom policy
402    Custom,
403}
404
405/// CORS configuration
406#[derive(Debug, Clone, Serialize, Deserialize)]
407pub struct CorsConfig {
408    /// Allowed origins
409    pub allowed_origins: Vec<String>,
410
411    /// Allowed methods
412    pub allowed_methods: Vec<String>,
413
414    /// Allowed headers
415    pub allowed_headers: Vec<String>,
416
417    /// Max age in seconds
418    pub max_age: Option<u32>,
419}
420
421/// Storage file representation
422#[derive(Debug, Clone, Serialize, Deserialize)]
423pub struct StorageFile {
424    /// Unique file identifier
425    pub id: String,
426
427    /// File name/path
428    pub name: String,
429
430    /// File size in bytes
431    pub size: u64,
432
433    /// MIME type
434    pub content_type: String,
435
436    /// File hash/checksum
437    pub checksum: Option<String>,
438
439    /// Checksum algorithm
440    pub checksum_algorithm: Option<String>,
441
442    /// File metadata
443    pub metadata: HashMap<String, String>,
444
445    /// Creation timestamp
446    pub created_at: DateTime<Utc>,
447
448    /// Last modification timestamp
449    pub updated_at: Option<DateTime<Utc>>,
450
451    /// File version (if versioning enabled)
452    pub version: Option<String>,
453
454    /// Storage backend
455    pub backend: StorageBackend,
456
457    /// Bucket/container name
458    pub bucket: String,
459
460    /// File path in storage
461    pub storage_path: String,
462
463    /// Whether file is encrypted
464    pub encrypted: bool,
465
466    /// Whether file is compressed
467    pub compressed: bool,
468
469    /// Expiration time (if set)
470    pub expires_at: Option<DateTime<Utc>>,
471}
472
473impl StorageFile {
474    /// Create new storage file
475    pub fn new(
476        name: String,
477        size: u64,
478        content_type: String,
479        bucket: String,
480        storage_path: String,
481        backend: StorageBackend,
482    ) -> Self {
483        Self {
484            id: Uuid::new_v4().to_string(),
485            name,
486            size,
487            content_type,
488            checksum: None,
489            checksum_algorithm: None,
490            metadata: HashMap::new(),
491            created_at: Utc::now(),
492            updated_at: None,
493            version: None,
494            backend,
495            bucket,
496            storage_path,
497            encrypted: false,
498            compressed: false,
499            expires_at: None,
500        }
501    }
502
503    /// Add metadata
504    pub fn with_metadata(mut self, key: String, value: String) -> Self {
505        self.metadata.insert(key, value);
506        self
507    }
508
509    /// Set checksum
510    pub fn with_checksum(mut self, checksum: String, algorithm: String) -> Self {
511        self.checksum = Some(checksum);
512        self.checksum_algorithm = Some(algorithm);
513        self
514    }
515
516    /// Set version
517    pub fn with_version(mut self, version: String) -> Self {
518        self.version = Some(version);
519        self
520    }
521
522    /// Set encryption
523    pub fn with_encryption(mut self, encrypted: bool) -> Self {
524        self.encrypted = encrypted;
525        self
526    }
527
528    /// Set compression
529    pub fn with_compression(mut self, compressed: bool) -> Self {
530        self.compressed = compressed;
531        self
532    }
533
534    /// Set expiration
535    pub fn with_expiration(mut self, expires_at: DateTime<Utc>) -> Self {
536        self.expires_at = Some(expires_at);
537        self
538    }
539
540    /// Set created at timestamp
541    pub fn with_created_at(mut self, created_at: DateTime<Utc>) -> Self {
542        self.created_at = created_at;
543        self
544    }
545
546    /// Set updated at timestamp
547    pub fn with_updated_at(mut self, updated_at: DateTime<Utc>) -> Self {
548        self.updated_at = Some(updated_at);
549        self
550    }
551
552    /// Check if file is expired
553    pub fn is_expired(&self) -> bool {
554        if let Some(expires_at) = self.expires_at {
555            Utc::now() > expires_at
556        } else {
557            false
558        }
559    }
560
561    /// Get file extension
562    pub fn extension(&self) -> Option<&str> {
563        std::path::Path::new(&self.name)
564            .extension()
565            .and_then(|ext| ext.to_str())
566    }
567}
568
569/// Upload options
570#[derive(Debug, Clone, Default)]
571pub struct UploadOptions {
572    /// Content type override
573    pub content_type: Option<String>,
574
575    /// File metadata
576    pub metadata: HashMap<String, String>,
577
578    /// Enable multipart upload for large files
579    pub multipart: Option<bool>,
580
581    /// Chunk size for multipart upload
582    pub chunk_size: Option<usize>,
583
584    /// Calculate checksum during upload
585    pub calculate_checksum: bool,
586
587    /// Checksum algorithm
588    pub checksum_algorithm: Option<String>,
589
590    /// Upload with encryption
591    pub encrypt: bool,
592
593    /// Upload with compression
594    pub compress: bool,
595
596    // Progress callback (removed for simplicity)
597    // pub progress_callback: Option<Box<dyn Fn(u64, u64) + Send + Sync>>,
598
599    /// Retry configuration
600    pub retry_config: Option<RetryConfig>,
601}
602
603/// Download options
604#[derive(Debug, Clone, Default)]
605pub struct DownloadOptions {
606    /// Byte range for download
607    pub range: Option<ByteRange>,
608
609    /// Verify checksum after download
610    pub verify_checksum: bool,
611
612    /// Expected checksum
613    pub expected_checksum: Option<String>,
614
615    // Progress callback (removed for simplicity)
616    // pub progress_callback: Option<Box<dyn Fn(u64, u64) + Send + Sync>>,
617
618    /// Retry configuration
619    pub retry_config: Option<RetryConfig>,
620}
621
622/// Byte range for partial downloads
623#[derive(Debug, Clone, Copy, PartialEq, Eq)]
624pub struct ByteRange {
625    /// Start byte position
626    pub start: u64,
627
628    /// End byte position (inclusive, or None for end of file)
629    pub end: Option<u64>,
630}
631
632impl ByteRange {
633    /// Create new byte range
634    pub fn new(start: u64, end: Option<u64>) -> Self {
635        Self { start, end }
636    }
637
638    /// Create range from start byte to end of file
639    pub fn from_start(start: u64) -> Self {
640        Self { start, end: None }
641    }
642
643    /// Create range for last N bytes
644    pub fn last_bytes(bytes: u64) -> Self {
645        Self {
646            start: 0,
647            end: Some(bytes - 1)
648        }
649    }
650
651    /// Convert to HTTP range header value
652    pub fn to_header_value(&self) -> String {
653        match self.end {
654            Some(end) => format!("bytes={}-{}", self.start, end),
655            None => format!("bytes={}-", self.start),
656        }
657    }
658}
659
660/// Presigned URL options
661#[derive(Debug, Clone)]
662pub struct PresignedUrlOptions {
663    /// Expiration time in seconds
664    pub expiry_seconds: u64,
665
666    /// HTTP method (GET, PUT, DELETE)
667    pub method: HttpMethod,
668
669    /// Custom headers to include
670    pub headers: HashMap<String, String>,
671
672    /// Query parameters to include
673    pub query_params: HashMap<String, String>,
674}
675
676impl Default for PresignedUrlOptions {
677    fn default() -> Self {
678        Self {
679            expiry_seconds: crate::DEFAULT_PRESIGNED_EXPIRY,
680            method: HttpMethod::GET,
681            headers: HashMap::new(),
682            query_params: HashMap::new(),
683        }
684    }
685}
686
687/// HTTP methods for presigned URLs
688#[derive(Debug, Clone, Copy, PartialEq, Eq)]
689pub enum HttpMethod {
690    GET,
691    PUT,
692    DELETE,
693    HEAD,
694}
695
696impl std::fmt::Display for HttpMethod {
697    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
698        match self {
699            Self::GET => write!(f, "GET"),
700            Self::PUT => write!(f, "PUT"),
701            Self::DELETE => write!(f, "DELETE"),
702            Self::HEAD => write!(f, "HEAD"),
703        }
704    }
705}
706
707/// Retry configuration
708#[derive(Debug, Clone)]
709pub struct RetryConfig {
710    /// Maximum number of retry attempts
711    pub max_attempts: u32,
712
713    /// Initial delay in milliseconds
714    pub initial_delay_ms: u64,
715
716    /// Backoff multiplier
717    pub backoff_multiplier: f64,
718
719    /// Maximum delay in milliseconds
720    pub max_delay_ms: u64,
721
722    /// Retry only on these errors
723    pub retry_on_error: Option<Vec<String>>,
724}
725
726impl Default for RetryConfig {
727    fn default() -> Self {
728        Self {
729            max_attempts: 3,
730            initial_delay_ms: 1000,
731            backoff_multiplier: 2.0,
732            max_delay_ms: 30000,
733            retry_on_error: None,
734        }
735    }
736}
737
738/// File metadata
739#[derive(Debug, Clone, Serialize, Deserialize)]
740pub struct FileMetadata {
741    /// File size in bytes
742    pub size: u64,
743
744    /// Last modification timestamp
745    pub last_modified: DateTime<Utc>,
746
747    /// Content type
748    pub content_type: String,
749
750    /// ETag
751    pub etag: Option<String>,
752
753    /// Storage class
754    pub storage_class: Option<String>,
755
756    /// Additional metadata
757    pub custom_metadata: HashMap<String, String>,
758}
759
760/// Storage statistics
761#[derive(Debug, Clone, Serialize, Deserialize)]
762pub struct StorageStats {
763    /// Total number of files
764    pub file_count: u64,
765
766    /// Total storage used in bytes
767    pub total_bytes: u64,
768
769    /// Storage used by each backend
770    pub backend_usage: HashMap<String, u64>,
771
772    /// Number of files by content type
773    pub content_type_counts: HashMap<String, u64>,
774
775    /// Average file size
776    pub average_file_size: f64,
777
778    /// Largest file size
779    pub largest_file_size: u64,
780
781    /// Last updated timestamp
782    pub last_updated: DateTime<Utc>,
783}
backbone_storage/types.rs

backbone_storage/
types.rs
