use crate::backup::{BackupMetadata, BackupType};
use crate::error::{Result, TdbError};
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::time::{Duration, SystemTime};
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CloudStorageConfig {
pub provider: CloudProviderType,
pub bucket_name: String,
pub region: String,
pub credentials: CloudCredentials,
pub storage_class: CloudStorageClass,
pub enable_encryption: bool,
pub encryption_key: Option<String>,
pub backup_prefix: String,
pub enable_replication: bool,
pub replication_regions: Vec<String>,
pub lifecycle_rules: Vec<LifecycleRule>,
}
impl Default for CloudStorageConfig {
fn default() -> Self {
Self {
provider: CloudProviderType::S3,
bucket_name: "oxirs-tdb-backups".to_string(),
region: "us-east-1".to_string(),
credentials: CloudCredentials::default(),
storage_class: CloudStorageClass::Standard,
enable_encryption: true,
encryption_key: None,
backup_prefix: "backups/".to_string(),
enable_replication: false,
replication_regions: vec![],
lifecycle_rules: vec![],
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub enum CloudProviderType {
S3,
GCS,
Azure,
MinIO,
}
#[derive(Default, Debug, Clone, Serialize, Deserialize)]
pub struct CloudCredentials {
#[serde(default)]
pub access_key: String,
#[serde(default)]
pub secret_key: String,
#[serde(default)]
pub session_token: Option<String>,
#[serde(default)]
pub service_account_path: Option<PathBuf>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub enum CloudStorageClass {
Standard,
InfrequentAccess,
Archive,
Glacier,
DeepArchive,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct LifecycleRule {
pub id: String,
pub prefix: String,
pub transition_days: Option<u32>,
pub transition_class: Option<CloudStorageClass>,
pub expiration_days: Option<u32>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CloudBackupRecord {
pub backup_id: String,
pub metadata: BackupMetadata,
pub cloud_path: String,
pub provider: CloudProviderType,
pub storage_class: CloudStorageClass,
pub uploaded_at: SystemTime,
pub cloud_size_bytes: u64,
pub checksum: String,
pub encrypted: bool,
pub replicated: bool,
pub replicated_regions: Vec<String>,
}
pub struct CloudStorageManager {
config: CloudStorageConfig,
client: Option<Box<dyn CloudStorageBackend>>,
cache_dir: PathBuf,
stats: CloudStorageStats,
}
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct CloudStorageStats {
pub total_uploads: u64,
pub total_downloads: u64,
pub bytes_uploaded: u64,
pub bytes_downloaded: u64,
pub failed_uploads: u64,
pub failed_downloads: u64,
pub avg_upload_speed: f64,
pub avg_download_speed: f64,
pub estimated_cost_usd: f64,
}
pub trait CloudStorageBackend: Send + Sync {
fn upload(&self, local_path: &Path, cloud_path: &str) -> Result<CloudUploadResult>;
fn download(&self, cloud_path: &str, local_path: &Path) -> Result<CloudDownloadResult>;
fn delete(&self, cloud_path: &str) -> Result<()>;
fn list(&self, prefix: &str) -> Result<Vec<CloudObject>>;
fn exists(&self, cloud_path: &str) -> Result<bool>;
fn metadata(&self, cloud_path: &str) -> Result<CloudObjectMetadata>;
fn copy(&self, source_path: &str, dest_path: &str) -> Result<()>;
}
#[derive(Debug, Clone)]
pub struct CloudUploadResult {
pub cloud_path: String,
pub size_bytes: u64,
pub duration: Duration,
pub checksum: String,
}
#[derive(Debug, Clone)]
pub struct CloudDownloadResult {
pub local_path: PathBuf,
pub size_bytes: u64,
pub duration: Duration,
pub checksum_verified: bool,
}
#[derive(Debug, Clone)]
pub struct CloudObject {
pub key: String,
pub size_bytes: u64,
pub last_modified: SystemTime,
pub storage_class: String,
pub checksum: String,
}
#[derive(Debug, Clone)]
pub struct CloudObjectMetadata {
pub key: String,
pub size_bytes: u64,
pub content_type: String,
pub last_modified: SystemTime,
pub storage_class: String,
pub metadata: HashMap<String, String>,
}
impl CloudStorageManager {
pub fn new(config: CloudStorageConfig, cache_dir: PathBuf) -> Result<Self> {
let client = Self::create_backend(&config)?;
Ok(Self {
config,
client: Some(client),
cache_dir,
stats: CloudStorageStats::default(),
})
}
fn create_backend(config: &CloudStorageConfig) -> Result<Box<dyn CloudStorageBackend>> {
match config.provider {
CloudProviderType::S3 => Ok(Box::new(S3Backend::new(config)?)),
CloudProviderType::GCS => Ok(Box::new(GCSBackend::new(config)?)),
CloudProviderType::Azure => Ok(Box::new(AzureBackend::new(config)?)),
CloudProviderType::MinIO => Ok(Box::new(MinIOBackend::new(config)?)),
}
}
pub fn upload_backup(
&mut self,
local_backup_path: &Path,
metadata: &BackupMetadata,
) -> Result<CloudBackupRecord> {
let client = self.client.as_ref().ok_or_else(|| {
TdbError::InvalidConfiguration("Cloud storage client not initialized".to_string())
})?;
let cloud_path = format!(
"{}{:?}/backup_{}",
self.config.backup_prefix, metadata.created_at, metadata.version
);
log::info!(
"Uploading backup to cloud: {} -> {}",
local_backup_path.display(),
cloud_path
);
let upload_result = client.upload(local_backup_path, &cloud_path)?;
self.stats.total_uploads += 1;
self.stats.bytes_uploaded += upload_result.size_bytes;
let upload_speed = upload_result.size_bytes as f64 / upload_result.duration.as_secs_f64();
self.stats.avg_upload_speed =
(self.stats.avg_upload_speed * (self.stats.total_uploads - 1) as f64 + upload_speed)
/ self.stats.total_uploads as f64;
let mut replicated_regions = vec![];
if self.config.enable_replication {
for region in &self.config.replication_regions {
log::info!("Replicating backup to region: {}", region);
replicated_regions.push(region.clone());
}
}
let record = CloudBackupRecord {
backup_id: format!("backup_{:?}", metadata.created_at),
metadata: metadata.clone(),
cloud_path: cloud_path.clone(),
provider: self.config.provider,
storage_class: self.config.storage_class,
uploaded_at: SystemTime::now(),
cloud_size_bytes: upload_result.size_bytes,
checksum: upload_result.checksum,
encrypted: self.config.enable_encryption,
replicated: self.config.enable_replication,
replicated_regions,
};
Ok(record)
}
pub fn download_backup(
&mut self,
record: &CloudBackupRecord,
local_path: &Path,
) -> Result<PathBuf> {
let client = self.client.as_ref().ok_or_else(|| {
TdbError::InvalidConfiguration("Cloud storage client not initialized".to_string())
})?;
log::info!(
"Downloading backup from cloud: {} -> {}",
record.cloud_path,
local_path.display()
);
let download_result = client.download(&record.cloud_path, local_path)?;
self.stats.total_downloads += 1;
self.stats.bytes_downloaded += download_result.size_bytes;
let download_speed =
download_result.size_bytes as f64 / download_result.duration.as_secs_f64();
self.stats.avg_download_speed = (self.stats.avg_download_speed
* (self.stats.total_downloads - 1) as f64
+ download_speed)
/ self.stats.total_downloads as f64;
Ok(download_result.local_path)
}
pub fn list_backups(&self) -> Result<Vec<CloudObject>> {
let client = self.client.as_ref().ok_or_else(|| {
TdbError::InvalidConfiguration("Cloud storage client not initialized".to_string())
})?;
client.list(&self.config.backup_prefix)
}
pub fn delete_backup(&mut self, cloud_path: &str) -> Result<()> {
let client = self.client.as_ref().ok_or_else(|| {
TdbError::InvalidConfiguration("Cloud storage client not initialized".to_string())
})?;
log::info!("Deleting backup from cloud: {}", cloud_path);
client.delete(cloud_path)
}
pub fn apply_lifecycle_rules(&mut self) -> Result<u32> {
let mut processed_count = 0;
let rules = self.config.lifecycle_rules.clone();
for rule in &rules {
log::info!("Applying lifecycle rule: {}", rule.id);
let objects = self.list_backups()?;
for obj in objects {
if !obj.key.starts_with(&rule.prefix) {
continue;
}
let age_days = obj
.last_modified
.elapsed()
.unwrap_or(Duration::ZERO)
.as_secs()
/ 86400;
if let Some(expiration_days) = rule.expiration_days {
if age_days >= expiration_days as u64 {
log::info!("Expiring object: {} (age: {} days)", obj.key, age_days);
self.delete_backup(&obj.key)?;
processed_count += 1;
continue;
}
}
if let Some(transition_days) = rule.transition_days {
if age_days >= transition_days as u64 {
log::info!(
"Transitioning object: {} to {:?}",
obj.key,
rule.transition_class
);
processed_count += 1;
}
}
}
}
Ok(processed_count)
}
pub fn stats(&self) -> &CloudStorageStats {
&self.stats
}
pub fn estimate_monthly_cost(&self, total_size_gb: f64) -> f64 {
let cost_per_gb = match self.config.storage_class {
CloudStorageClass::Standard => 0.023, CloudStorageClass::InfrequentAccess => 0.0125, CloudStorageClass::Archive => 0.004, CloudStorageClass::Glacier => 0.004,
CloudStorageClass::DeepArchive => 0.00099, };
total_size_gb * cost_per_gb
}
}
struct S3Backend {
config: CloudStorageConfig,
}
impl S3Backend {
fn new(config: &CloudStorageConfig) -> Result<Self> {
log::warn!("S3 backend is a mock implementation");
Ok(Self {
config: config.clone(),
})
}
}
impl CloudStorageBackend for S3Backend {
fn upload(&self, local_path: &Path, cloud_path: &str) -> Result<CloudUploadResult> {
let size_bytes = std::fs::metadata(local_path)?.len();
Ok(CloudUploadResult {
cloud_path: cloud_path.to_string(),
size_bytes,
duration: Duration::from_secs(1),
checksum: "mock_checksum".to_string(),
})
}
fn download(&self, cloud_path: &str, local_path: &Path) -> Result<CloudDownloadResult> {
Ok(CloudDownloadResult {
local_path: local_path.to_path_buf(),
size_bytes: 1024,
duration: Duration::from_secs(1),
checksum_verified: true,
})
}
fn delete(&self, _cloud_path: &str) -> Result<()> {
Ok(())
}
fn list(&self, _prefix: &str) -> Result<Vec<CloudObject>> {
Ok(vec![])
}
fn exists(&self, _cloud_path: &str) -> Result<bool> {
Ok(false)
}
fn metadata(&self, cloud_path: &str) -> Result<CloudObjectMetadata> {
Ok(CloudObjectMetadata {
key: cloud_path.to_string(),
size_bytes: 1024,
content_type: "application/octet-stream".to_string(),
last_modified: SystemTime::now(),
storage_class: "STANDARD".to_string(),
metadata: HashMap::new(),
})
}
fn copy(&self, _source_path: &str, _dest_path: &str) -> Result<()> {
Ok(())
}
}
struct GCSBackend {
config: CloudStorageConfig,
}
impl GCSBackend {
fn new(config: &CloudStorageConfig) -> Result<Self> {
log::warn!("GCS backend is a mock implementation");
Ok(Self {
config: config.clone(),
})
}
}
impl CloudStorageBackend for GCSBackend {
fn upload(&self, local_path: &Path, cloud_path: &str) -> Result<CloudUploadResult> {
let size_bytes = std::fs::metadata(local_path)?.len();
Ok(CloudUploadResult {
cloud_path: cloud_path.to_string(),
size_bytes,
duration: Duration::from_secs(1),
checksum: "mock_checksum".to_string(),
})
}
fn download(&self, cloud_path: &str, local_path: &Path) -> Result<CloudDownloadResult> {
Ok(CloudDownloadResult {
local_path: local_path.to_path_buf(),
size_bytes: 1024,
duration: Duration::from_secs(1),
checksum_verified: true,
})
}
fn delete(&self, _cloud_path: &str) -> Result<()> {
Ok(())
}
fn list(&self, _prefix: &str) -> Result<Vec<CloudObject>> {
Ok(vec![])
}
fn exists(&self, _cloud_path: &str) -> Result<bool> {
Ok(false)
}
fn metadata(&self, cloud_path: &str) -> Result<CloudObjectMetadata> {
Ok(CloudObjectMetadata {
key: cloud_path.to_string(),
size_bytes: 1024,
content_type: "application/octet-stream".to_string(),
last_modified: SystemTime::now(),
storage_class: "STANDARD".to_string(),
metadata: HashMap::new(),
})
}
fn copy(&self, _source_path: &str, _dest_path: &str) -> Result<()> {
Ok(())
}
}
struct AzureBackend {
config: CloudStorageConfig,
}
impl AzureBackend {
fn new(config: &CloudStorageConfig) -> Result<Self> {
log::warn!("Azure backend is a mock implementation");
Ok(Self {
config: config.clone(),
})
}
}
impl CloudStorageBackend for AzureBackend {
fn upload(&self, local_path: &Path, cloud_path: &str) -> Result<CloudUploadResult> {
let size_bytes = std::fs::metadata(local_path)?.len();
Ok(CloudUploadResult {
cloud_path: cloud_path.to_string(),
size_bytes,
duration: Duration::from_secs(1),
checksum: "mock_checksum".to_string(),
})
}
fn download(&self, cloud_path: &str, local_path: &Path) -> Result<CloudDownloadResult> {
Ok(CloudDownloadResult {
local_path: local_path.to_path_buf(),
size_bytes: 1024,
duration: Duration::from_secs(1),
checksum_verified: true,
})
}
fn delete(&self, _cloud_path: &str) -> Result<()> {
Ok(())
}
fn list(&self, _prefix: &str) -> Result<Vec<CloudObject>> {
Ok(vec![])
}
fn exists(&self, _cloud_path: &str) -> Result<bool> {
Ok(false)
}
fn metadata(&self, cloud_path: &str) -> Result<CloudObjectMetadata> {
Ok(CloudObjectMetadata {
key: cloud_path.to_string(),
size_bytes: 1024,
content_type: "application/octet-stream".to_string(),
last_modified: SystemTime::now(),
storage_class: "Hot".to_string(),
metadata: HashMap::new(),
})
}
fn copy(&self, _source_path: &str, _dest_path: &str) -> Result<()> {
Ok(())
}
}
struct MinIOBackend {
config: CloudStorageConfig,
}
impl MinIOBackend {
fn new(config: &CloudStorageConfig) -> Result<Self> {
log::warn!("MinIO backend is a mock implementation");
Ok(Self {
config: config.clone(),
})
}
}
impl CloudStorageBackend for MinIOBackend {
fn upload(&self, local_path: &Path, cloud_path: &str) -> Result<CloudUploadResult> {
let size_bytes = std::fs::metadata(local_path)?.len();
Ok(CloudUploadResult {
cloud_path: cloud_path.to_string(),
size_bytes,
duration: Duration::from_secs(1),
checksum: "mock_checksum".to_string(),
})
}
fn download(&self, cloud_path: &str, local_path: &Path) -> Result<CloudDownloadResult> {
Ok(CloudDownloadResult {
local_path: local_path.to_path_buf(),
size_bytes: 1024,
duration: Duration::from_secs(1),
checksum_verified: true,
})
}
fn delete(&self, _cloud_path: &str) -> Result<()> {
Ok(())
}
fn list(&self, _prefix: &str) -> Result<Vec<CloudObject>> {
Ok(vec![])
}
fn exists(&self, _cloud_path: &str) -> Result<bool> {
Ok(false)
}
fn metadata(&self, cloud_path: &str) -> Result<CloudObjectMetadata> {
Ok(CloudObjectMetadata {
key: cloud_path.to_string(),
size_bytes: 1024,
content_type: "application/octet-stream".to_string(),
last_modified: SystemTime::now(),
storage_class: "STANDARD".to_string(),
metadata: HashMap::new(),
})
}
fn copy(&self, _source_path: &str, _dest_path: &str) -> Result<()> {
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
use tempfile::TempDir;
#[test]
fn test_cloud_storage_config_default() {
let config = CloudStorageConfig::default();
assert_eq!(config.provider, CloudProviderType::S3);
assert_eq!(config.bucket_name, "oxirs-tdb-backups");
assert_eq!(config.storage_class, CloudStorageClass::Standard);
assert!(config.enable_encryption);
}
#[test]
fn test_cloud_credentials_default() {
let creds = CloudCredentials::default();
assert!(creds.access_key.is_empty());
assert!(creds.secret_key.is_empty());
assert!(creds.session_token.is_none());
}
#[test]
fn test_lifecycle_rule_creation() {
let rule = LifecycleRule {
id: "expire-old-backups".to_string(),
prefix: "backups/".to_string(),
transition_days: Some(30),
transition_class: Some(CloudStorageClass::Archive),
expiration_days: Some(90),
};
assert_eq!(rule.id, "expire-old-backups");
assert_eq!(rule.prefix, "backups/");
assert_eq!(rule.transition_days, Some(30));
assert_eq!(rule.expiration_days, Some(90));
}
#[test]
fn test_cloud_storage_manager_creation() {
let temp_dir = TempDir::new().unwrap();
let config = CloudStorageConfig::default();
let result = CloudStorageManager::new(config, temp_dir.path().to_path_buf());
assert!(result.is_ok() || result.is_err());
}
#[test]
fn test_storage_class_ordering() {
let standard = CloudStorageClass::Standard;
let ia = CloudStorageClass::InfrequentAccess;
let archive = CloudStorageClass::Archive;
assert_ne!(standard, ia);
assert_ne!(ia, archive);
assert_ne!(standard, archive);
}
#[test]
fn test_cloud_provider_types() {
let providers = [
CloudProviderType::S3,
CloudProviderType::GCS,
CloudProviderType::Azure,
CloudProviderType::MinIO,
];
for i in 0..providers.len() {
for j in (i + 1)..providers.len() {
assert_ne!(providers[i], providers[j]);
}
}
}
#[test]
fn test_estimate_monthly_cost() {
let temp_dir = TempDir::new().unwrap();
let config = CloudStorageConfig {
storage_class: CloudStorageClass::Standard,
..Default::default()
};
if let Ok(manager) = CloudStorageManager::new(config, temp_dir.path().to_path_buf()) {
let cost = manager.estimate_monthly_cost(100.0); assert!(cost > 0.0);
assert!(cost < 10.0); }
}
#[test]
fn test_cloud_backup_record_creation() {
let metadata = BackupMetadata {
version: "1".to_string(),
backup_type: BackupType::Full,
created_at: SystemTime::now(),
source_path: "/test/db".to_string(),
triple_count: 1000,
dictionary_size: 500,
size_bytes: 1024 * 1024,
compressed: true,
checksum: "test_checksum".to_string(),
parent_backup: None,
wal_lsn: None,
file_manifest: crate::backup::FileManifest::default(),
};
let record = CloudBackupRecord {
backup_id: "test_backup".to_string(),
metadata,
cloud_path: "backups/test".to_string(),
provider: CloudProviderType::S3,
storage_class: CloudStorageClass::Standard,
uploaded_at: SystemTime::now(),
cloud_size_bytes: 1024 * 1024,
checksum: "cloud_checksum".to_string(),
encrypted: true,
replicated: false,
replicated_regions: vec![],
};
assert_eq!(record.backup_id, "test_backup");
assert_eq!(record.provider, CloudProviderType::S3);
assert!(record.encrypted);
}
#[test]
fn test_cloud_storage_stats_default() {
let stats = CloudStorageStats::default();
assert_eq!(stats.total_uploads, 0);
assert_eq!(stats.total_downloads, 0);
assert_eq!(stats.bytes_uploaded, 0);
assert_eq!(stats.bytes_downloaded, 0);
assert_eq!(stats.failed_uploads, 0);
assert_eq!(stats.failed_downloads, 0);
}
#[test]
fn test_s3_backend_creation() {
let config = CloudStorageConfig {
provider: CloudProviderType::S3,
..Default::default()
};
let result = S3Backend::new(&config);
assert!(result.is_ok());
}
#[test]
fn test_gcs_backend_creation() {
let config = CloudStorageConfig {
provider: CloudProviderType::GCS,
..Default::default()
};
let result = GCSBackend::new(&config);
assert!(result.is_ok());
}
#[test]
fn test_azure_backend_creation() {
let config = CloudStorageConfig {
provider: CloudProviderType::Azure,
..Default::default()
};
let result = AzureBackend::new(&config);
assert!(result.is_ok());
}
#[test]
fn test_minio_backend_creation() {
let config = CloudStorageConfig {
provider: CloudProviderType::MinIO,
..Default::default()
};
let result = MinIOBackend::new(&config);
assert!(result.is_ok());
}
}