Skip to main content

commonware_runtime/storage/
faulty.rs

1//! A storage wrapper that injects deterministic faults for testing crash recovery.
2
3use crate::{deterministic::BoxDynRng, Error, Handle, IoBufs, IoBufsMut};
4use bytes::Buf;
5use commonware_utils::sync::{Mutex, RwLock};
6use rand::RngExt as _;
7use std::{
8    io::Error as IoError,
9    sync::{
10        atomic::{AtomicU64, Ordering},
11        Arc,
12    },
13};
14
15/// Operation types for fault injection.
16#[derive(Debug, Clone, Copy, PartialEq, Eq)]
17enum Op {
18    Open,
19    Read,
20    Write,
21    Sync,
22    Resize,
23    Remove,
24    Scan,
25}
26
27/// Configuration for deterministic storage fault injection.
28///
29/// Each rate is a probability from 0.0 (never fail) to 1.0 (always fail).
30#[derive(Clone, Debug, Default)]
31pub struct Config {
32    /// Failure rate for `open_versioned` operations.
33    pub open_rate: Option<f64>,
34
35    /// Failure rate for `read_at` operations.
36    pub read_rate: Option<f64>,
37
38    /// Failure rate for `write_at` operations.
39    pub write_rate: Option<f64>,
40
41    /// Probability that a write failure is a partial write (some bytes written
42    /// before failure) rather than a complete failure (no bytes written).
43    /// Only applies when `write_rate` triggers a failure.
44    /// Value from 0.0 (always complete failure) to 1.0 (always partial write).
45    pub partial_write_rate: Option<f64>,
46
47    /// Failure rate for `sync` operations.
48    pub sync_rate: Option<f64>,
49
50    /// Failure rate for `resize` operations.
51    pub resize_rate: Option<f64>,
52
53    /// Probability that a resize failure is partial (resized to an intermediate
54    /// size before failure) rather than a complete failure (size unchanged).
55    /// Only applies when `resize_rate` triggers a failure.
56    /// Value from 0.0 (always complete failure) to 1.0 (always partial resize).
57    pub partial_resize_rate: Option<f64>,
58
59    /// Failure rate for `remove` operations.
60    pub remove_rate: Option<f64>,
61
62    /// Failure rate for `scan` operations.
63    pub scan_rate: Option<f64>,
64}
65
66impl Config {
67    /// Get the failure rate for an operation type.
68    fn rate_for(&self, op: Op) -> f64 {
69        match op {
70            Op::Open => self.open_rate,
71            Op::Read => self.read_rate,
72            Op::Write => self.write_rate,
73            Op::Sync => self.sync_rate,
74            Op::Resize => self.resize_rate,
75            Op::Remove => self.remove_rate,
76            Op::Scan => self.scan_rate,
77        }
78        .unwrap_or(0.0)
79    }
80
81    /// Set the open failure rate.
82    pub const fn open(mut self, rate: f64) -> Self {
83        self.open_rate = Some(rate);
84        self
85    }
86
87    /// Set the read failure rate.
88    pub const fn read(mut self, rate: f64) -> Self {
89        self.read_rate = Some(rate);
90        self
91    }
92
93    /// Set the write failure rate.
94    pub const fn write(mut self, rate: f64) -> Self {
95        self.write_rate = Some(rate);
96        self
97    }
98
99    /// Set the partial write rate (probability of partial vs complete write failure).
100    pub const fn partial_write(mut self, rate: f64) -> Self {
101        self.partial_write_rate = Some(rate);
102        self
103    }
104
105    /// Set the sync failure rate.
106    pub const fn sync(mut self, rate: f64) -> Self {
107        self.sync_rate = Some(rate);
108        self
109    }
110
111    /// Set the resize failure rate.
112    pub const fn resize(mut self, rate: f64) -> Self {
113        self.resize_rate = Some(rate);
114        self
115    }
116
117    /// Set the partial resize rate (probability of partial vs complete resize failure).
118    pub const fn partial_resize(mut self, rate: f64) -> Self {
119        self.partial_resize_rate = Some(rate);
120        self
121    }
122
123    /// Set the remove failure rate.
124    pub const fn remove(mut self, rate: f64) -> Self {
125        self.remove_rate = Some(rate);
126        self
127    }
128
129    /// Set the scan failure rate.
130    pub const fn scan(mut self, rate: f64) -> Self {
131        self.scan_rate = Some(rate);
132        self
133    }
134}
135
136/// Shared fault injection context.
137#[derive(Clone)]
138struct Oracle {
139    rng: Arc<Mutex<BoxDynRng>>,
140    config: Arc<RwLock<Config>>,
141}
142
143impl Oracle {
144    /// Check if a fault should be injected for the given operation.
145    fn should_fail(&self, op: Op) -> bool {
146        self.roll(Some(self.config.read().rate_for(op)))
147    }
148
149    /// Check if a write fault should be injected. Returns (should_fail, partial_rate).
150    /// Reads config once to avoid nested lock acquisition.
151    fn check_write_fault(&self) -> (bool, Option<f64>) {
152        let config = self.config.read();
153        let fail = self.roll(Some(config.rate_for(Op::Write)));
154        (fail, config.partial_write_rate)
155    }
156
157    /// Check if a resize fault should be injected. Returns (should_fail, partial_rate).
158    /// Reads config once to avoid nested lock acquisition.
159    fn check_resize_fault(&self) -> (bool, Option<f64>) {
160        let config = self.config.read();
161        let fail = self.roll(Some(config.rate_for(Op::Resize)));
162        (fail, config.partial_resize_rate)
163    }
164
165    /// Check if an event should occur based on a probability rate.
166    fn roll(&self, rate: Option<f64>) -> bool {
167        let rate = rate.unwrap_or(0.0);
168        if rate <= 0.0 {
169            return false;
170        }
171        if rate >= 1.0 {
172            return true;
173        }
174        self.rng.lock().random::<f64>() < rate
175    }
176
177    /// Generate a random value strictly between `from` and `to`, or None if not possible.
178    fn random_between(&self, from: u64, to: u64) -> Option<u64> {
179        if from == to {
180            return None;
181        }
182        let (min, max) = if from < to { (from, to) } else { (to, from) };
183        if max - min <= 1 {
184            return None;
185        }
186        Some(self.rng.lock().random_range(min + 1..max))
187    }
188
189    /// Try to generate a partial operation target. Returns Some if both the rate
190    /// check passes and an intermediate value exists between `from` and `to`.
191    fn try_partial(&self, rate: Option<f64>, from: u64, to: u64) -> Option<u64> {
192        if self.roll(rate) {
193            self.random_between(from, to)
194        } else {
195            None
196        }
197    }
198}
199
200/// A storage wrapper that injects deterministic faults based on configuration.
201///
202/// Uses a shared RNG for determinism.
203#[derive(Clone)]
204pub struct Storage<S: crate::Storage> {
205    inner: S,
206    ctx: Oracle,
207}
208
209impl<S: crate::Storage> Storage<S> {
210    /// Create a new faulty storage wrapper.
211    pub fn new(inner: S, rng: Arc<Mutex<BoxDynRng>>, config: Arc<RwLock<Config>>) -> Self {
212        Self {
213            inner,
214            ctx: Oracle { rng, config },
215        }
216    }
217
218    /// Get a reference to the inner storage.
219    pub const fn inner(&self) -> &S {
220        &self.inner
221    }
222
223    /// Get access to the fault configuration for dynamic modification.
224    pub fn config(&self) -> Arc<RwLock<Config>> {
225        self.ctx.config.clone()
226    }
227}
228
229/// Create an IoError for injected faults.
230fn injected_io_error() -> IoError {
231    IoError::other("injected storage fault")
232}
233
234impl<S: crate::Storage> crate::Storage for Storage<S> {
235    type Blob = Blob<S::Blob>;
236
237    async fn open_versioned(
238        &self,
239        partition: &str,
240        name: &[u8],
241        versions: std::ops::RangeInclusive<u16>,
242    ) -> Result<(Self::Blob, u64, u16), Error> {
243        if self.ctx.should_fail(Op::Open) {
244            return Err(injected_io_error().into());
245        }
246        self.inner
247            .open_versioned(partition, name, versions)
248            .await
249            .map(|(blob, len, blob_version)| {
250                (Blob::new(self.ctx.clone(), blob, len), len, blob_version)
251            })
252    }
253
254    async fn remove(&self, partition: &str, name: Option<&[u8]>) -> Result<(), Error> {
255        if self.ctx.should_fail(Op::Remove) {
256            return Err(injected_io_error().into());
257        }
258        self.inner.remove(partition, name).await
259    }
260
261    async fn scan(&self, partition: &str) -> Result<Vec<Vec<u8>>, Error> {
262        if self.ctx.should_fail(Op::Scan) {
263            return Err(injected_io_error().into());
264        }
265        self.inner.scan(partition).await
266    }
267}
268
269/// A blob wrapper that injects deterministic faults based on configuration.
270#[derive(Clone)]
271pub struct Blob<B: crate::Blob> {
272    inner: B,
273    ctx: Oracle,
274    /// Tracked size for partial resize support.
275    size: Arc<AtomicU64>,
276}
277
278impl<B: crate::Blob> Blob<B> {
279    fn new(ctx: Oracle, inner: B, size: u64) -> Self {
280        Self {
281            inner,
282            ctx,
283            size: Arc::new(AtomicU64::new(size)),
284        }
285    }
286}
287
288impl<B: crate::Blob> crate::Blob for Blob<B> {
289    async fn read_at(&self, offset: u64, len: usize) -> Result<IoBufsMut, Error> {
290        if self.ctx.should_fail(Op::Read) {
291            return Err(injected_io_error().into());
292        }
293        self.inner.read_at(offset, len).await
294    }
295
296    async fn read_at_buf(
297        &self,
298        offset: u64,
299        len: usize,
300        bufs: impl Into<IoBufsMut> + Send,
301    ) -> Result<IoBufsMut, Error> {
302        if self.ctx.should_fail(Op::Read) {
303            return Err(injected_io_error().into());
304        }
305        self.inner.read_at_buf(offset, len, bufs.into()).await
306    }
307
308    async fn write_at(&self, offset: u64, bufs: impl Into<IoBufs> + Send) -> Result<(), Error> {
309        let bufs = bufs.into();
310        let total_bytes = bufs.remaining() as u64;
311
312        let (should_fail, partial_rate) = self.ctx.check_write_fault();
313        if should_fail {
314            if let Some(bytes) = self.ctx.try_partial(partial_rate, 0, total_bytes) {
315                // Partial write: write some bytes, sync, then fail
316                self.inner
317                    .write_at(offset, bufs.coalesce().slice(..bytes as usize))
318                    .await?;
319                self.inner.sync().await?;
320                self.size
321                    .fetch_max(offset.saturating_add(bytes), Ordering::Relaxed);
322                return Err(injected_io_error().into());
323            }
324            return Err(injected_io_error().into());
325        }
326
327        self.inner.write_at(offset, bufs).await?;
328        self.size
329            .fetch_max(offset.saturating_add(total_bytes), Ordering::Relaxed);
330        Ok(())
331    }
332
333    async fn write_at_sync(
334        &self,
335        offset: u64,
336        bufs: impl Into<IoBufs> + Send,
337    ) -> Result<(), Error> {
338        let bufs = bufs.into();
339        let total_bytes = bufs.remaining() as u64;
340        if total_bytes == 0 {
341            return Ok(());
342        }
343
344        let (should_fail, partial_rate) = self.ctx.check_write_fault();
345        if should_fail {
346            if let Some(bytes) = self.ctx.try_partial(partial_rate, 0, total_bytes) {
347                self.inner
348                    .write_at_sync(offset, bufs.coalesce().slice(..bytes as usize))
349                    .await?;
350                self.size
351                    .fetch_max(offset.saturating_add(bytes), Ordering::Relaxed);
352                return Err(injected_io_error().into());
353            }
354            return Err(injected_io_error().into());
355        }
356
357        if self.ctx.should_fail(Op::Sync) {
358            self.inner.write_at(offset, bufs).await?;
359            self.size
360                .fetch_max(offset.saturating_add(total_bytes), Ordering::Relaxed);
361            return Err(injected_io_error().into());
362        }
363
364        self.inner.write_at_sync(offset, bufs).await?;
365        self.size
366            .fetch_max(offset.saturating_add(total_bytes), Ordering::Relaxed);
367        Ok(())
368    }
369
370    async fn resize(&self, len: u64) -> Result<(), Error> {
371        let (should_fail, partial_rate) = self.ctx.check_resize_fault();
372        if should_fail {
373            let current = self.size.load(Ordering::Relaxed);
374            if let Some(len) = self.ctx.try_partial(partial_rate, current, len) {
375                // Partial resize: resize to intermediate size, sync, then fail
376                self.inner.resize(len).await?;
377                self.inner.sync().await?;
378                self.size.store(len, Ordering::Relaxed);
379                return Err(injected_io_error().into());
380            }
381            return Err(injected_io_error().into());
382        }
383        self.inner.resize(len).await?;
384        self.size.store(len, Ordering::Relaxed);
385        Ok(())
386    }
387
388    async fn sync(&self) -> Result<(), Error> {
389        if self.ctx.should_fail(Op::Sync) {
390            return Err(injected_io_error().into());
391        }
392        self.inner.sync().await
393    }
394
395    async fn start_sync(&self) -> Handle<()> {
396        if self.ctx.should_fail(Op::Sync) {
397            return Handle::ready(Err(injected_io_error().into()));
398        }
399        self.inner.start_sync().await
400    }
401}
402
403#[cfg(test)]
404mod tests {
405    use super::*;
406    use crate::{
407        storage::{memory::Storage as MemStorage, tests::run_storage_tests},
408        telemetry::metrics::Registry,
409        Blob as _, BufferPool, BufferPoolConfig, Storage as _,
410    };
411    use rand::{rngs::StdRng, SeedableRng};
412
413    fn test_pool() -> BufferPool {
414        let mut registry = Registry::default();
415        BufferPool::new(BufferPoolConfig::for_storage(), &mut registry)
416    }
417
418    /// Test harness with faulty storage wrapping memory storage.
419    struct Harness {
420        inner: MemStorage,
421        storage: Storage<MemStorage>,
422        config: Arc<RwLock<Config>>,
423    }
424
425    impl Harness {
426        fn new(config: Config) -> Self {
427            Self::with_seed(42, config)
428        }
429
430        fn with_seed(seed: u64, config: Config) -> Self {
431            let inner = MemStorage::new(test_pool());
432            let rng = Arc::new(Mutex::new(
433                Box::new(StdRng::seed_from_u64(seed)) as BoxDynRng
434            ));
435            let config = Arc::new(RwLock::new(config));
436            let storage = Storage::new(inner.clone(), rng, config.clone());
437            Self {
438                inner,
439                storage,
440                config,
441            }
442        }
443    }
444
445    #[tokio::test]
446    async fn test_faulty_storage_no_faults() {
447        let h = Harness::new(Config::default());
448        run_storage_tests(h.storage).await;
449    }
450
451    #[tokio::test]
452    async fn test_faulty_storage_sync_always_fails() {
453        let h = Harness::new(Config::default().sync(1.0));
454
455        let (blob, _) = h.storage.open("partition", b"test").await.unwrap();
456        blob.write_at(0, b"data".to_vec()).await.unwrap();
457
458        assert!(matches!(blob.sync().await, Err(Error::Io(_))));
459    }
460
461    #[tokio::test]
462    async fn test_faulty_storage_start_sync_always_fails() {
463        let h = Harness::new(Config::default().sync(1.0));
464
465        let (blob, _) = h.storage.open("partition", b"test").await.unwrap();
466        blob.write_at(0, b"data".to_vec()).await.unwrap();
467
468        let result = blob.start_sync().await.await;
469        assert!(matches!(result, Err(Error::Io(_))));
470    }
471
472    #[tokio::test]
473    async fn test_faulty_storage_write_always_fails() {
474        let h = Harness::new(Config::default().write(1.0));
475
476        let (blob, _) = h.storage.open("partition", b"test").await.unwrap();
477
478        assert!(matches!(
479            blob.write_at(0, b"data".to_vec()).await,
480            Err(Error::Io(_))
481        ));
482    }
483
484    #[tokio::test]
485    async fn test_faulty_storage_write_at_sync_write_always_fails() {
486        let h = Harness::new(Config::default().write(1.0));
487
488        let (blob, _) = h.storage.open("partition", b"test").await.unwrap();
489
490        assert!(matches!(
491            blob.write_at_sync(0, b"data".to_vec()).await,
492            Err(Error::Io(_))
493        ));
494    }
495
496    #[tokio::test]
497    async fn test_faulty_storage_write_at_sync_sync_failure_is_not_durable() {
498        let h = Harness::new(Config::default().sync(1.0));
499
500        let (blob, _) = h.storage.open("partition", b"test").await.unwrap();
501
502        assert!(matches!(
503            blob.write_at_sync(0, b"data".to_vec()).await,
504            Err(Error::Io(_))
505        ));
506
507        let (_reopened, size) = h.inner.open("partition", b"test").await.unwrap();
508        assert_eq!(size, 0);
509    }
510
511    #[tokio::test]
512    async fn test_faulty_storage_empty_write_at_sync_does_not_sync_prior_write() {
513        let h = Harness::new(Config::default().sync(1.0));
514
515        let (blob, _) = h.storage.open("partition", b"test").await.unwrap();
516        blob.write_at(0, b"data".to_vec()).await.unwrap();
517
518        blob.write_at_sync(4, Vec::<u8>::new()).await.unwrap();
519
520        let (_reopened, size) = h.inner.open("partition", b"test").await.unwrap();
521        assert_eq!(size, 0);
522    }
523
524    #[tokio::test]
525    async fn test_faulty_storage_read_always_fails() {
526        let h = Harness::new(Config::default());
527
528        // Write some data first (no faults)
529        let (blob, _) = h.storage.open("partition", b"test").await.unwrap();
530        blob.write_at(0, b"data".to_vec()).await.unwrap();
531        blob.sync().await.unwrap();
532
533        // Enable read faults
534        h.config.write().read_rate = Some(1.0);
535
536        assert!(matches!(blob.read_at(0, 4).await, Err(Error::Io(_))));
537    }
538
539    #[tokio::test]
540    async fn test_faulty_storage_open_always_fails() {
541        let h = Harness::new(Config::default().open(1.0));
542
543        assert!(matches!(
544            h.storage.open("partition", b"test").await,
545            Err(Error::Io(_))
546        ));
547    }
548
549    #[tokio::test]
550    async fn test_faulty_storage_remove_always_fails() {
551        let h = Harness::new(Config::default());
552
553        // Create a blob first
554        let (blob, _) = h.storage.open("partition", b"test").await.unwrap();
555        blob.write_at(0, b"data".to_vec()).await.unwrap();
556        blob.sync().await.unwrap();
557        drop(blob);
558
559        // Enable remove faults
560        h.config.write().remove_rate = Some(1.0);
561
562        assert!(matches!(
563            h.storage.remove("partition", Some(b"test")).await,
564            Err(Error::Io(_))
565        ));
566    }
567
568    #[tokio::test]
569    async fn test_faulty_storage_scan_always_fails() {
570        let h = Harness::new(Config::default());
571
572        // Create some blobs first
573        for i in 0..3 {
574            let name = format!("blob{i}");
575            let (blob, _) = h.storage.open("partition", name.as_bytes()).await.unwrap();
576            blob.write_at(0, b"data".to_vec()).await.unwrap();
577            blob.sync().await.unwrap();
578        }
579
580        // Enable scan faults
581        h.config.write().scan_rate = Some(1.0);
582
583        assert!(matches!(
584            h.storage.scan("partition").await,
585            Err(Error::Io(_))
586        ));
587    }
588
589    #[tokio::test]
590    async fn test_faulty_storage_determinism() {
591        async fn run_ops(seed: u64, rate: f64) -> Vec<bool> {
592            let h = Harness::with_seed(seed, Config::default().open(rate));
593            let mut results = Vec::new();
594            for i in 0..20 {
595                let name = format!("blob{i}");
596                results.push(h.storage.open("partition", name.as_bytes()).await.is_ok());
597            }
598            results
599        }
600
601        let results1 = run_ops(42, 0.5).await;
602        let results2 = run_ops(42, 0.5).await;
603        assert_eq!(results1, results2, "Same seed should produce same results");
604
605        let results3 = run_ops(999, 0.5).await;
606        assert_ne!(
607            results1, results3,
608            "Different seeds should produce different results"
609        );
610    }
611
612    #[tokio::test]
613    async fn test_faulty_storage_rate_for() {
614        let config = Config::default().open(0.1).sync(0.9);
615
616        assert!((config.rate_for(Op::Open) - 0.1).abs() < f64::EPSILON);
617        assert!((config.rate_for(Op::Sync) - 0.9).abs() < f64::EPSILON);
618        assert!(config.rate_for(Op::Write).abs() < f64::EPSILON);
619    }
620
621    #[tokio::test]
622    async fn test_faulty_storage_dynamic_config() {
623        let h = Harness::new(Config::default());
624
625        let (blob, _) = h.storage.open("partition", b"test").await.unwrap();
626        blob.sync().await.unwrap();
627
628        h.config.write().sync_rate = Some(1.0);
629        assert!(matches!(blob.sync().await, Err(Error::Io(_))));
630
631        h.config.write().sync_rate = Some(0.0);
632        blob.sync().await.unwrap();
633    }
634
635    #[tokio::test]
636    async fn test_faulty_storage_partial_write() {
637        let h = Harness::new(Config::default().write(1.0).partial_write(1.0));
638
639        let (blob, _) = h.storage.open("partition", b"test").await.unwrap();
640        let data = b"hello world".to_vec();
641        let result = blob.write_at(0, data.clone()).await;
642
643        assert!(matches!(result, Err(Error::Io(_))));
644
645        let (inner_blob, size) = h.inner.open("partition", b"test").await.unwrap();
646        let bytes_written = size as usize;
647        assert!(
648            bytes_written > 0 && bytes_written < data.len(),
649            "Expected partial write: {bytes_written} bytes out of {}",
650            data.len()
651        );
652
653        let read_result = inner_blob.read_at(0, bytes_written).await.unwrap();
654        assert_eq!(read_result.coalesce().as_ref(), &data[..bytes_written]);
655    }
656
657    #[tokio::test]
658    async fn test_faulty_storage_partial_write_disabled() {
659        let h = Harness::new(Config::default().write(1.0).partial_write(0.0));
660
661        let (blob, _) = h.storage.open("partition", b"test").await.unwrap();
662        let result = blob.write_at(0, b"hello world".to_vec()).await;
663
664        assert!(matches!(result, Err(Error::Io(_))));
665
666        let (_, size) = h.inner.open("partition", b"test").await.unwrap();
667        assert_eq!(
668            size, 0,
669            "Expected no bytes written when partial_write_rate is 0"
670        );
671    }
672
673    #[tokio::test]
674    async fn test_faulty_storage_partial_write_single_byte() {
675        let h = Harness::new(Config::default().write(1.0).partial_write(1.0));
676
677        let (blob, _) = h.storage.open("partition", b"test").await.unwrap();
678        let result = blob.write_at(0, b"x".to_vec()).await;
679
680        assert!(matches!(result, Err(Error::Io(_))));
681
682        let (_, size) = h.inner.open("partition", b"test").await.unwrap();
683        assert_eq!(size, 0, "No partial write possible for single byte");
684    }
685
686    #[tokio::test]
687    async fn test_faulty_storage_partial_resize_grow() {
688        let h = Harness::new(Config::default().resize(1.0).partial_resize(1.0));
689
690        let (blob, initial_size) = h.storage.open("partition", b"test").await.unwrap();
691        assert_eq!(initial_size, 0);
692
693        let target_size = 100u64;
694        let result = blob.resize(target_size).await;
695
696        assert!(matches!(result, Err(Error::Io(_))));
697
698        let (_, actual_size) = h.inner.open("partition", b"test").await.unwrap();
699        assert!(
700            actual_size > 0 && actual_size < target_size,
701            "Expected partial resize: size {actual_size} should be between 0 and {target_size}"
702        );
703    }
704
705    #[tokio::test]
706    async fn test_faulty_storage_partial_resize_shrink() {
707        let h = Harness::new(Config::default());
708
709        let (blob, _) = h.storage.open("partition", b"test").await.unwrap();
710        blob.resize(100).await.unwrap();
711        blob.sync().await.unwrap();
712
713        {
714            let mut cfg = h.config.write();
715            cfg.resize_rate = Some(1.0);
716            cfg.partial_resize_rate = Some(1.0);
717        }
718
719        let target_size = 10u64;
720        let result = blob.resize(target_size).await;
721
722        assert!(matches!(result, Err(Error::Io(_))));
723
724        let (_, actual_size) = h.inner.open("partition", b"test").await.unwrap();
725        assert!(
726            actual_size > target_size && actual_size < 100,
727            "Expected partial shrink: size {actual_size} should be between {target_size} and 100"
728        );
729    }
730
731    #[tokio::test]
732    async fn test_faulty_storage_partial_resize_disabled() {
733        let h = Harness::new(Config::default().resize(1.0).partial_resize(0.0));
734
735        let (blob, _) = h.storage.open("partition", b"test").await.unwrap();
736        let result = blob.resize(100).await;
737
738        assert!(matches!(result, Err(Error::Io(_))));
739
740        let (_, size) = h.inner.open("partition", b"test").await.unwrap();
741        assert_eq!(size, 0, "Expected no resize when partial_resize_rate is 0");
742    }
743
744    #[tokio::test]
745    async fn test_faulty_storage_partial_resize_same_size() {
746        let h = Harness::new(Config::default().resize(1.0).partial_resize(1.0));
747
748        let (blob, _) = h.storage.open("partition", b"test").await.unwrap();
749        let result = blob.resize(0).await;
750
751        assert!(matches!(result, Err(Error::Io(_))));
752
753        let (_, size) = h.inner.open("partition", b"test").await.unwrap();
754        assert_eq!(size, 0);
755    }
756
757    #[tokio::test]
758    async fn test_faulty_storage_partial_resize_after_write_extends() {
759        let h = Harness::new(Config::default());
760
761        let (blob, initial_size) = h.storage.open("partition", b"test").await.unwrap();
762        assert_eq!(initial_size, 0);
763
764        blob.write_at(0, vec![0xABu8; 50]).await.unwrap();
765        blob.sync().await.unwrap();
766
767        let (_, size_after_write) = h.inner.open("partition", b"test").await.unwrap();
768        assert_eq!(size_after_write, 50);
769
770        {
771            let mut cfg = h.config.write();
772            cfg.resize_rate = Some(1.0);
773            cfg.partial_resize_rate = Some(1.0);
774        }
775
776        let target_size = 10u64;
777        let result = blob.resize(target_size).await;
778
779        assert!(matches!(result, Err(Error::Io(_))));
780
781        let (_, actual_size) = h.inner.open("partition", b"test").await.unwrap();
782        assert!(
783            actual_size > target_size && actual_size < 50,
784            "Expected partial shrink from 50: size {actual_size} should be between {target_size} and 50"
785        );
786    }
787
788    #[tokio::test]
789    async fn test_faulty_storage_partial_resize_one_byte_difference() {
790        let h = Harness::new(Config::default().resize(1.0).partial_resize(1.0));
791
792        let (blob, _) = h.storage.open("partition", b"test").await.unwrap();
793        let result = blob.resize(1).await;
794
795        assert!(matches!(result, Err(Error::Io(_))));
796
797        let (_, size) = h.inner.open("partition", b"test").await.unwrap();
798        assert_eq!(size, 0);
799    }
800}