sherpack_kube/storage/
mod.rs

1//! Storage drivers for persisting release information
2//!
3//! Sherpack supports multiple storage backends:
4//! - **Secrets** (default): Store releases in Kubernetes Secrets (like Helm)
5//! - **ConfigMap**: Store releases in ConfigMaps (less secure, but more accessible)
6//! - **File**: Store releases in local files (for development/testing)
7//!
8//! ## Key Improvements over Helm
9//!
10//! - **Zstd compression**: Better compression ratio than gzip (~30% smaller)
11//! - **Large release handling**: Automatic chunking or external storage for >1MB releases
12//! - **JSON format**: Human-readable after decompression (vs Helm's protobuf)
13
14mod chunked;
15mod configmap;
16mod file;
17mod mock;
18mod secrets;
19
20pub use chunked::{CHUNK_SIZE, ChunkedIndex, ChunkedStorage};
21pub use configmap::ConfigMapDriver;
22pub use file::FileDriver;
23pub use mock::{MockStorageDriver, OperationCounts};
24pub use secrets::SecretsDriver;
25
26use crate::error::{KubeError, Result};
27use crate::release::StoredRelease;
28use async_trait::async_trait;
29
30/// Maximum size for a single Kubernetes Secret/ConfigMap (1MB - some overhead)
31pub const MAX_RESOURCE_SIZE: usize = 1_000_000;
32
33/// Storage driver trait for release persistence
34///
35/// Implementations must be Send + Sync for use across async tasks.
36#[async_trait]
37pub trait StorageDriver: Send + Sync {
38    /// Get a specific release by name and version
39    async fn get(&self, namespace: &str, name: &str, version: u32) -> Result<StoredRelease>;
40
41    /// Get the latest release for a name
42    async fn get_latest(&self, namespace: &str, name: &str) -> Result<StoredRelease>;
43
44    /// List all releases, optionally filtered by namespace and/or name
45    async fn list(
46        &self,
47        namespace: Option<&str>,
48        name: Option<&str>,
49        include_superseded: bool,
50    ) -> Result<Vec<StoredRelease>>;
51
52    /// Get release history (all versions for a name)
53    async fn history(&self, namespace: &str, name: &str) -> Result<Vec<StoredRelease>>;
54
55    /// Create a new release
56    async fn create(&self, release: &StoredRelease) -> Result<()>;
57
58    /// Update an existing release
59    async fn update(&self, release: &StoredRelease) -> Result<()>;
60
61    /// Delete a specific release version
62    async fn delete(&self, namespace: &str, name: &str, version: u32) -> Result<StoredRelease>;
63
64    /// Delete all versions of a release
65    async fn delete_all(&self, namespace: &str, name: &str) -> Result<Vec<StoredRelease>>;
66
67    /// Check if a release exists
68    async fn exists(&self, namespace: &str, name: &str) -> Result<bool> {
69        match self.get_latest(namespace, name).await {
70            Ok(_) => Ok(true),
71            Err(KubeError::ReleaseNotFound { .. }) => Ok(false),
72            Err(e) => Err(e),
73        }
74    }
75}
76
77/// Storage configuration
78#[derive(Debug, Clone)]
79pub struct StorageConfig {
80    /// Compression method
81    pub compression: CompressionMethod,
82
83    /// Strategy for handling large releases
84    pub large_release_strategy: LargeReleaseStrategy,
85
86    /// Maximum number of revisions to keep per release
87    pub max_history: u32,
88}
89
90impl Default for StorageConfig {
91    fn default() -> Self {
92        Self {
93            compression: CompressionMethod::Zstd { level: 3 },
94            large_release_strategy: LargeReleaseStrategy::ChunkedSecrets,
95            max_history: 10,
96        }
97    }
98}
99
100/// Compression method for release data
101#[derive(Debug, Clone, Copy, PartialEq, Eq)]
102pub enum CompressionMethod {
103    /// No compression
104    None,
105
106    /// Gzip compression (Helm-compatible)
107    Gzip { level: u32 },
108
109    /// Zstd compression (better ratio, faster)
110    Zstd { level: i32 },
111}
112
113impl Default for CompressionMethod {
114    fn default() -> Self {
115        Self::Zstd { level: 3 }
116    }
117}
118
119/// Strategy for handling releases larger than MAX_RESOURCE_SIZE
120#[derive(Debug, Clone, PartialEq, Eq, Default)]
121pub enum LargeReleaseStrategy {
122    /// Fail if release is too large (Helm default behavior)
123    Fail,
124
125    /// Split across multiple Secrets/ConfigMaps
126    #[default]
127    ChunkedSecrets,
128
129    /// Store manifest separately in a ConfigMap
130    SeparateManifest,
131
132    /// Reference external storage (manifest stored elsewhere)
133    ExternalReference {
134        /// S3-compatible endpoint URL
135        endpoint: String,
136        /// Bucket name
137        bucket: String,
138    },
139}
140
141/// Compress data using the configured method
142#[must_use = "compression result should be used"]
143pub fn compress(data: &[u8], method: CompressionMethod) -> Result<Vec<u8>> {
144    match method {
145        CompressionMethod::None => Ok(data.to_vec()),
146        CompressionMethod::Gzip { level } => {
147            use std::io::Write;
148            let mut encoder =
149                flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::new(level));
150            encoder
151                .write_all(data)
152                .map_err(|e| KubeError::Compression(e.to_string()))?;
153            encoder
154                .finish()
155                .map_err(|e| KubeError::Compression(e.to_string()))
156        }
157        CompressionMethod::Zstd { level } => zstd::encode_all(std::io::Cursor::new(data), level)
158            .map_err(|e| KubeError::Compression(e.to_string())),
159    }
160}
161
162/// Decompress data
163#[must_use = "decompression result should be used"]
164pub fn decompress(data: &[u8], method: CompressionMethod) -> Result<Vec<u8>> {
165    match method {
166        CompressionMethod::None => Ok(data.to_vec()),
167        CompressionMethod::Gzip { .. } => {
168            use std::io::Read;
169            let mut decoder = flate2::read::GzDecoder::new(data);
170            let mut decompressed = Vec::new();
171            decoder
172                .read_to_end(&mut decompressed)
173                .map_err(|e| KubeError::Compression(e.to_string()))?;
174            Ok(decompressed)
175        }
176        CompressionMethod::Zstd { .. } => zstd::decode_all(std::io::Cursor::new(data))
177            .map_err(|e| KubeError::Compression(e.to_string())),
178    }
179}
180
181/// Serialize a release to JSON bytes
182#[must_use = "serialization result should be used"]
183pub fn serialize_release(release: &StoredRelease) -> Result<Vec<u8>> {
184    serde_json::to_vec(release).map_err(|e| KubeError::Serialization(e.to_string()))
185}
186
187/// Deserialize a release from JSON bytes
188#[must_use = "deserialization result should be used"]
189pub fn deserialize_release(data: &[u8]) -> Result<StoredRelease> {
190    serde_json::from_slice(data).map_err(|e| KubeError::Serialization(e.to_string()))
191}
192
193/// Encode data for storage (serialize + compress + base64)
194#[must_use = "encoded data should be used for storage"]
195pub fn encode_for_storage(release: &StoredRelease, config: &StorageConfig) -> Result<String> {
196    let json = serialize_release(release)?;
197    let compressed = compress(&json, config.compression)?;
198    Ok(base64::Engine::encode(
199        &base64::engine::general_purpose::STANDARD,
200        &compressed,
201    ))
202}
203
204/// Decode data from storage (base64 + decompress + deserialize)
205#[must_use = "decoded release should be used"]
206pub fn decode_from_storage(data: &str, compression: CompressionMethod) -> Result<StoredRelease> {
207    let decoded = base64::Engine::decode(&base64::engine::general_purpose::STANDARD, data)
208        .map_err(|e| KubeError::Serialization(format!("base64 decode error: {}", e)))?;
209    let decompressed = decompress(&decoded, compression)?;
210    deserialize_release(&decompressed)
211}
212
213/// Labels applied to all storage resources
214#[must_use = "labels should be applied to resources"]
215pub fn storage_labels(release: &StoredRelease) -> std::collections::BTreeMap<String, String> {
216    let mut labels = std::collections::BTreeMap::new();
217    labels.insert(
218        "app.kubernetes.io/managed-by".to_string(),
219        "sherpack".to_string(),
220    );
221    labels.insert("sherpack.io/release-name".to_string(), release.name.clone());
222    labels.insert(
223        "sherpack.io/release-version".to_string(),
224        release.version.to_string(),
225    );
226    labels.insert(
227        "sherpack.io/release-namespace".to_string(),
228        release.namespace.clone(),
229    );
230    labels
231}
232
233#[cfg(test)]
234mod tests {
235    use super::*;
236    use crate::release::ReleaseState;
237    use sherpack_core::{PackMetadata, Values};
238
239    fn test_release() -> StoredRelease {
240        StoredRelease::for_install(
241            "test".to_string(),
242            "default".to_string(),
243            PackMetadata {
244                name: "test-pack".to_string(),
245                version: semver::Version::new(1, 0, 0),
246                description: Some("Test pack".to_string()),
247                app_version: None,
248                kube_version: None,
249                home: None,
250                icon: None,
251                sources: vec![],
252                keywords: vec![],
253                maintainers: vec![],
254                annotations: Default::default(),
255            },
256            Values::new(),
257            "apiVersion: v1\nkind: ConfigMap".to_string(),
258        )
259    }
260
261    fn test_release_with_manifest(manifest: &str) -> StoredRelease {
262        StoredRelease::for_install(
263            "test".to_string(),
264            "default".to_string(),
265            PackMetadata {
266                name: "test-pack".to_string(),
267                version: semver::Version::new(1, 0, 0),
268                description: None,
269                app_version: None,
270                kube_version: None,
271                home: None,
272                icon: None,
273                sources: vec![],
274                keywords: vec![],
275                maintainers: vec![],
276                annotations: Default::default(),
277            },
278            Values::new(),
279            manifest.to_string(),
280        )
281    }
282
283    #[test]
284    fn test_compression_roundtrip_zstd() {
285        let data = b"Hello, World! This is test data for compression.";
286        let compressed = compress(data, CompressionMethod::Zstd { level: 3 }).unwrap();
287        let decompressed = decompress(&compressed, CompressionMethod::Zstd { level: 3 }).unwrap();
288        assert_eq!(data.as_slice(), decompressed.as_slice());
289    }
290
291    #[test]
292    fn test_compression_roundtrip_gzip() {
293        let data = b"Hello, World! This is test data for compression.";
294        let compressed = compress(data, CompressionMethod::Gzip { level: 6 }).unwrap();
295        let decompressed = decompress(&compressed, CompressionMethod::Gzip { level: 6 }).unwrap();
296        assert_eq!(data.as_slice(), decompressed.as_slice());
297    }
298
299    #[test]
300    fn test_compression_none() {
301        let data = b"No compression test data";
302        let compressed = compress(data, CompressionMethod::None).unwrap();
303        assert_eq!(data.as_slice(), compressed.as_slice());
304        let decompressed = decompress(&compressed, CompressionMethod::None).unwrap();
305        assert_eq!(data.as_slice(), decompressed.as_slice());
306    }
307
308    #[test]
309    fn test_encode_decode_roundtrip() {
310        let release = test_release();
311        let config = StorageConfig::default();
312
313        let encoded = encode_for_storage(&release, &config).unwrap();
314        let decoded = decode_from_storage(&encoded, config.compression).unwrap();
315
316        assert_eq!(release.name, decoded.name);
317        assert_eq!(release.namespace, decoded.namespace);
318        assert_eq!(release.version, decoded.version);
319    }
320
321    #[test]
322    fn test_encode_decode_no_compression() {
323        let release = test_release();
324        let config = StorageConfig {
325            compression: CompressionMethod::None,
326            ..Default::default()
327        };
328
329        let encoded = encode_for_storage(&release, &config).unwrap();
330        let decoded = decode_from_storage(&encoded, config.compression).unwrap();
331
332        assert_eq!(release.name, decoded.name);
333        assert_eq!(release.manifest, decoded.manifest);
334    }
335
336    #[test]
337    fn test_encode_decode_gzip() {
338        let release = test_release();
339        let config = StorageConfig {
340            compression: CompressionMethod::Gzip { level: 6 },
341            ..Default::default()
342        };
343
344        let encoded = encode_for_storage(&release, &config).unwrap();
345        let decoded = decode_from_storage(&encoded, config.compression).unwrap();
346
347        assert_eq!(release.name, decoded.name);
348    }
349
350    #[test]
351    fn test_zstd_smaller_than_gzip() {
352        // Large data to show compression difference
353        let data: Vec<u8> = (0..10000).map(|i| (i % 256) as u8).collect();
354
355        let zstd_compressed = compress(&data, CompressionMethod::Zstd { level: 3 }).unwrap();
356        let gzip_compressed = compress(&data, CompressionMethod::Gzip { level: 6 }).unwrap();
357
358        // Zstd should be smaller or similar
359        assert!(
360            zstd_compressed.len() <= gzip_compressed.len() + 100,
361            "Zstd: {}, Gzip: {}",
362            zstd_compressed.len(),
363            gzip_compressed.len()
364        );
365    }
366
367    #[test]
368    fn test_storage_labels() {
369        let release = test_release();
370        let labels = storage_labels(&release);
371
372        assert_eq!(
373            labels.get("app.kubernetes.io/managed-by"),
374            Some(&"sherpack".to_string())
375        );
376        assert_eq!(
377            labels.get("sherpack.io/release-name"),
378            Some(&"test".to_string())
379        );
380        assert_eq!(
381            labels.get("sherpack.io/release-version"),
382            Some(&"1".to_string())
383        );
384        assert_eq!(
385            labels.get("sherpack.io/release-namespace"),
386            Some(&"default".to_string())
387        );
388    }
389
390    #[test]
391    fn test_serialize_deserialize_release() {
392        let release = test_release();
393        let serialized = serialize_release(&release).unwrap();
394        let deserialized = deserialize_release(&serialized).unwrap();
395
396        assert_eq!(release.name, deserialized.name);
397        assert_eq!(release.namespace, deserialized.namespace);
398        assert_eq!(release.version, deserialized.version);
399        assert_eq!(release.manifest, deserialized.manifest);
400    }
401
402    #[test]
403    fn test_serialize_release_with_all_fields() {
404        let mut release = test_release();
405        release.notes = Some("Installation notes".to_string());
406        release.labels.insert("env".to_string(), "prod".to_string());
407
408        let serialized = serialize_release(&release).unwrap();
409        let deserialized = deserialize_release(&serialized).unwrap();
410
411        assert_eq!(deserialized.notes, Some("Installation notes".to_string()));
412        assert_eq!(deserialized.labels.get("env"), Some(&"prod".to_string()));
413    }
414
415    #[test]
416    fn test_storage_config_default() {
417        let config = StorageConfig::default();
418
419        assert!(matches!(
420            config.compression,
421            CompressionMethod::Zstd { level: 3 }
422        ));
423        assert!(matches!(
424            config.large_release_strategy,
425            LargeReleaseStrategy::ChunkedSecrets
426        ));
427        assert_eq!(config.max_history, 10);
428    }
429
430    #[test]
431    fn test_large_manifest_compression() {
432        // Create a release with a large manifest
433        let large_manifest = "apiVersion: v1\nkind: ConfigMap\n".repeat(1000);
434        let release = test_release_with_manifest(&large_manifest);
435        let config = StorageConfig::default();
436
437        let encoded = encode_for_storage(&release, &config).unwrap();
438        let decoded = decode_from_storage(&encoded, config.compression).unwrap();
439
440        assert_eq!(release.manifest, decoded.manifest);
441
442        // Compressed should be smaller than original JSON
443        let json = serialize_release(&release).unwrap();
444        let base64_decoded =
445            base64::Engine::decode(&base64::engine::general_purpose::STANDARD, &encoded).unwrap();
446        assert!(
447            base64_decoded.len() < json.len(),
448            "Compressed {} should be smaller than JSON {}",
449            base64_decoded.len(),
450            json.len()
451        );
452    }
453
454    #[test]
455    fn test_decode_invalid_base64() {
456        let result = decode_from_storage("not valid base64!!!", CompressionMethod::None);
457        assert!(result.is_err());
458    }
459
460    #[test]
461    fn test_decode_invalid_json() {
462        // Valid base64 but not valid JSON
463        let invalid =
464            base64::Engine::encode(&base64::engine::general_purpose::STANDARD, b"not json");
465        let result = decode_from_storage(&invalid, CompressionMethod::None);
466        assert!(result.is_err());
467    }
468
469    #[test]
470    fn test_release_state_preserved() {
471        let mut release = test_release();
472        release.state = ReleaseState::Failed {
473            reason: "Test failure".to_string(),
474            recoverable: true,
475            failed_at: chrono::Utc::now(),
476        };
477
478        let serialized = serialize_release(&release).unwrap();
479        let deserialized = deserialize_release(&serialized).unwrap();
480
481        assert!(
482            matches!(deserialized.state, ReleaseState::Failed { reason, .. } if reason == "Test failure")
483        );
484    }
485}
sherpack_kube/storage/mod.rs

sherpack_kube/storage/
mod.rs
