store-io-engine 0.5.1

The store-io engine: volumes, regions, barriers with flush sharing, receipts, provisioning and A/B slot objects over any store-io platform.
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
//! Region handles: [`AppendRegion`], [`PageRegion`] and [`Slot`].
//!
//! Every handle is cheap to clone and safe to share between threads. Calls
//! block until their I/O is done.

use std::ops::ControlFlow;
use std::sync::Arc;
use std::sync::atomic::Ordering;

use store_io_core::error::{Error, FirstCause, Named, NotWrittenCause};
use store_io_core::id::Generation;
use store_io_format::meta::RegionKind;
use store_io_platform::Platform;

use crate::batch::{AppendBatch, PageBatch};
use crate::gate::Pass;
use crate::io::{ctx, ctx_range, not_written};
use crate::lifecycle::retired_error;
use crate::mapped::Mapped;
use crate::options::RegionOptions;
use crate::receipt::{DurableReceipt, RegionPos, WriteTicket};
use crate::scan::{ScanItem, ScanSummary};
use crate::settle::{Wrote, settle_pumped};
use crate::store::{LiveRegion, Store};

pub use crate::slot_handle::Slot;

/// Region lookups and provisioning.
impl<P: Platform> Store<P>
where
    P::Queue: Send,
{
    /// Looks up an existing append region.
    ///
    /// # Errors
    ///
    /// [`Error::NotFound`] if no ready append region has that name.
    pub fn append_region(&self, name: &str) -> Result<AppendRegion<P>, Error> {
        let state = self.find(name, RegionKind::Append, Named::Region)?;
        Ok(AppendRegion {
            store: self.clone(),
            state,
        })
    }

    /// Provisions a new append region of at least `size` bytes.
    ///
    /// # Errors
    ///
    /// [`Error::AlreadyExists`], [`Error::NoSpace`], or a platform error.
    pub fn provision_append_region(&self, name: &str, size: u64) -> Result<AppendRegion<P>, Error> {
        self.provision_append_region_with(name, RegionOptions::default().size(size))
    }

    /// Provisions a new append region as `opts` say (its size, and in
    /// container format 2 its cap).
    ///
    /// # Errors
    ///
    /// As [`Self::provision_append_region`]; `NotWritten(InvalidOptions)`
    /// for options [`RegionOptions::validate`] refuses; `Unsupported` for a
    /// cap in container format 1.
    pub fn provision_append_region_with(
        &self,
        name: &str,
        opts: RegionOptions,
    ) -> Result<AppendRegion<P>, Error> {
        let state = self.provision(name, RegionKind::Append, opts, None)?;
        Ok(AppendRegion {
            store: self.clone(),
            state,
        })
    }

    /// Provisions a new append region with the store's
    /// [`StoreOptions::region_defaults`](crate::StoreOptions::region_defaults).
    ///
    /// ```
    /// # use std::path::Path;
    /// # use store_io_engine::{RegionOptions, Store, StoreOptions};
    /// # use store_io_sim::{SimConfig, SimPlatform};
    /// let opts = StoreOptions {
    ///     region_defaults: Some(RegionOptions::default().size(1 << 20)),
    ///     ..StoreOptions::default()
    /// };
    /// let store = Store::create(SimPlatform::new(SimConfig::volatile(1)), Path::new("/db"), opts)?;
    /// let log = store.provision_append_region_default("log")?;
    /// assert_eq!(log.len(), 1 << 20);
    /// # Ok::<(), store_io_core::error::Error>(())
    /// ```
    ///
    /// # Errors
    ///
    /// As [`Self::provision_append_region_with`];
    /// `NotWritten(InvalidOptions)` when the store has no defaults (before
    /// any other refusal).
    pub fn provision_append_region_default(&self, name: &str) -> Result<AppendRegion<P>, Error> {
        self.provision_append_region_with(name, self.region_defaults()?)
    }

    /// Provisions a new page region with the store's
    /// [`StoreOptions::region_defaults`](crate::StoreOptions::region_defaults).
    ///
    /// # Errors
    ///
    /// As [`Self::provision_append_region_default`].
    pub fn provision_page_region_default(&self, name: &str) -> Result<PageRegion<P>, Error> {
        self.provision_page_region_with(name, self.region_defaults()?)
    }

    /// The store's region defaults; refused when it has none (before any
    /// I/O, and before the store's own refusals: read-only, poisoned).
    pub(crate) fn region_defaults(&self) -> Result<RegionOptions, Error> {
        self.inner.opts.region_defaults.ok_or(Error::NotWritten {
            cause: NotWrittenCause::InvalidOptions,
            ctx: store_io_core::error::ErrorContext::default(),
        })
    }

    /// Looks up an existing page region.
    ///
    /// # Errors
    ///
    /// [`Error::NotFound`] if no ready page region has that name.
    pub fn page_region(&self, name: &str) -> Result<PageRegion<P>, Error> {
        let state = self.find(name, RegionKind::Page, Named::Region)?;
        Ok(PageRegion {
            store: self.clone(),
            state,
        })
    }

    /// Provisions a new page region of at least `size` bytes.
    ///
    /// # Errors
    ///
    /// [`Error::AlreadyExists`], [`Error::NoSpace`], or a platform error.
    pub fn provision_page_region(&self, name: &str, size: u64) -> Result<PageRegion<P>, Error> {
        self.provision_page_region_with(name, RegionOptions::default().size(size))
    }

    /// Provisions a new page region as `opts` say (its size, and in
    /// container format 2 its cap).
    ///
    /// # Errors
    ///
    /// As [`Self::provision_append_region_with`].
    pub fn provision_page_region_with(
        &self,
        name: &str,
        opts: RegionOptions,
    ) -> Result<PageRegion<P>, Error> {
        let state = self.provision(name, RegionKind::Page, opts, None)?;
        Ok(PageRegion {
            store: self.clone(),
            state,
        })
    }

    /// Looks up an existing small-object slot.
    ///
    /// # Errors
    ///
    /// [`Error::NotFound`] if no slot has that name.
    pub fn slot(&self, name: &str) -> Result<Slot<P>, Error> {
        let state = self.find(name, RegionKind::Slot, Named::Slot)?;
        Ok(Slot::from_parts(self.clone(), state))
    }

    /// Provisions a new small-object slot. Its capacity is the block size
    /// minus 176 bytes of headers ([`Slot::capacity`]).
    ///
    /// # Errors
    ///
    /// [`Error::AlreadyExists`], [`Error::NoSpace`], or a platform error.
    pub fn provision_slot(&self, name: &str) -> Result<Slot<P>, Error> {
        let state = self.provision(name, RegionKind::Slot, RegionOptions::default(), None)?;
        Ok(Slot::from_parts(self.clone(), state))
    }
}

// ---------------------------------------------------------------------------
// Append regions
// ---------------------------------------------------------------------------

/// A region store-io appends to. Positions are chosen by store-io from an
/// atomic tail, so appends from many threads never overlap and a written
/// block is never rewritten.
///
/// Each append occupies whole blocks: the bytes from the end of the data to
/// the next block boundary are zero and are not caller data. Pack small
/// records into one append (or use a batch) to avoid the padding.
pub struct AppendRegion<P: Platform> {
    store: Store<P>,
    state: Arc<LiveRegion>,
}

impl<P: Platform> AppendRegion<P> {
    pub(crate) fn from_parts(store: Store<P>, state: Arc<LiveRegion>) -> Self {
        Self { store, state }
    }
}

impl<P: Platform> Clone for AppendRegion<P> {
    fn clone(&self) -> Self {
        Self {
            store: self.store.clone(),
            state: Arc::clone(&self.state),
        }
    }
}

impl<P: Platform> AppendRegion<P>
where
    P::Queue: Send,
{
    /// Threads waiting on the region's completed prefix (an append waiting
    /// for room in the window, a sync waiting for its appends). For tests.
    #[doc(hidden)]
    #[must_use]
    pub fn waiting_on_appends(&self) -> u64 {
        self.state.frontier.as_ref().map_or(0, |f| f.waiting())
    }

    fn frontier(&self) -> Result<&crate::frontier::AppendFrontier, Error> {
        self.state
            .frontier
            .as_ref()
            .ok_or_else(|| not_written(NotWrittenCause::NotReady, ctx(&self.state)))
    }

    /// The checks every append makes before touching the device, and the
    /// pass that keeps a recycle or release out until the append is done.
    pub(crate) fn writable(&self) -> Result<(&crate::frontier::AppendFrontier, Pass<'_>), Error> {
        let pass = self.pass()?;
        self.store.ready_for(&self.state)?;
        let f = self.frontier()?;
        if !self.state.positioned.load(Ordering::Acquire) {
            return Err(not_written(
                NotWrittenCause::NotPositioned,
                ctx(&self.state),
            ));
        }
        Ok((f, pass))
    }

    fn pass(&self) -> Result<Pass<'_>, Error> {
        self.state
            .gate
            .enter()
            .map_err(|why| retired_error(why, &self.state))
    }

    /// The generation this handle writes and reads. Recycling starts the
    /// next one; embed it in your records to tell generations apart after a
    /// crash.
    #[must_use]
    pub fn generation(&self) -> Generation {
        self.state.generation
    }

    /// Starts a new generation: waits for every operation in flight on this
    /// region, rewrites the region header with the next generation, makes it
    /// durable, and returns a handle to the new generation with its append
    /// position at 0. No other I/O: the old bytes stay on disk, so records
    /// must carry their generation. Every handle, ticket and position of the
    /// old generation is refused from then on (`StaleGeneration`).
    ///
    /// Must not be called from inside this region's own scan visitor, nor by
    /// a thread whose own [`crate::Submitter`] has operations on this region
    /// it has not reaped: recycle waits for them, and only that thread's
    /// `reap` or `wait` finishes them (the same holds for `release`).
    ///
    /// # Errors
    ///
    /// `NotWritten(StaleGeneration | NotReady)` if the region was already
    /// recycled or released; `NotWritten(ReadOnly)` if the store or the
    /// region is read-only; `Io` if the store's queue could not be prepared
    /// (nothing was written; the region stays usable); `Poisoned`;
    /// `DurabilityUnknown`.
    pub fn recycle(&self) -> Result<Self, Error> {
        let state = self.store.recycle_region(&self.state)?;
        Ok(Self {
            store: self.store.clone(),
            state,
        })
    }

    /// Releases the region: waits for every operation in flight, records
    /// the release in the region table (durable: from then on the release
    /// stands), marks its header released (durable), and gives its space
    /// back to the file system. Its contents are undefined from then on;
    /// every handle to it is refused (`NotReady`), and its space is reused
    /// by later provisioning. A header write that fails before it reaches
    /// the device (refused, a queue that could not be prepared, a header
    /// pair that would not encode or plan) still returns success: the space
    /// is free for provisioning, but not given back to the file system. Not
    /// from a thread with unreaped [`crate::Submitter`] operations on it
    /// (see [`Self::recycle`]).
    ///
    /// # Errors
    ///
    /// `NotWritten(StaleGeneration | NotReady)` if the region was already
    /// recycled or released; `NotWritten(ReadOnly)` if the store or the
    /// region is read-only; on format 2, `NotWritten(Frozen)` while a table
    /// page is stale or lost, and `NotWritten(Overlap)` when removing the
    /// entry would change another region's verdict at the next open;
    /// `Poisoned`; `DurabilityUnknown`; or `Io` if the space could not be
    /// deallocated (the region is released either way). Once the table
    /// records the release, a release fails only on a poisoned store or
    /// with that `Io`.
    pub fn release(self) -> Result<(), Error> {
        self.store.release_region(&self.state)
    }

    pub(crate) fn store(&self) -> &Store<P> {
        &self.store
    }

    pub(crate) fn state(&self) -> &Arc<LiveRegion> {
        &self.state
    }

    /// Appends `data` (any length of at least one byte) and returns a ticket
    /// once the write has completed. The data is not yet durable: pass the
    /// ticket to [`Self::sync_through`], or use [`Self::append_durable`].
    ///
    /// The append starts at a block boundary chosen by store-io and is
    /// zero-padded to the next one. Data larger than the largest pooled
    /// buffer is written in several pieces, kept in flight together.
    ///
    /// # Errors
    ///
    /// `NotWritten` (empty data, region full, pool exhausted, read-only, not
    /// positioned), `Io` if the store's queue could not be prepared (nothing
    /// was reserved or written), `Poisoned`, or `DurabilityUnknown` (the
    /// domain is now poisoned).
    pub fn append(&self, data: &[u8]) -> Result<WriteTicket, Error> {
        let s = &self.store;
        let (f, _pass) = self.writable()?;
        let c = ctx(&self.state);
        if data.is_empty() {
            return Err(not_written(NotWrittenCause::Empty, c));
        }
        // Everything that can fail without touching the device happens first:
        // the first piece is staged before any space is reserved.
        let mut stream = s.stage(data, c)?;
        let (mut lane, reserved) = s.lane_and_reservation(&self.state, f, data.len() as u64, c)?;
        let r = reserved.range();
        stream.at(r.start);
        // Until the pump returns the guard poisons on any exit (a panic
        // included); then the settle step completes the reservation, or
        // poisons: past it every failure leaves a hole.
        let outcome = s.pump(&mut lane, &mut Mapped::new(&self.state.map, &mut stream));
        let r = reserved.disarm();
        settle_pumped(s, &self.state, Wrote::Append(r), outcome)
    }

    /// Makes everything appended up to and including `ticket` durable.
    ///
    /// # Errors
    ///
    /// `NotWritten(ForeignPosition | StaleGeneration)` for a ticket of another
    /// region or generation; `Poisoned`; `DurabilityUnknown` if the flush
    /// failed.
    pub fn sync_through(&self, ticket: &WriteTicket) -> Result<DurableReceipt, Error> {
        let s = &self.store;
        let f = self.frontier()?;
        let c = ctx(&self.state);
        self.state
            .gate
            .check()
            .map_err(|why| retired_error(why, &self.state))?;
        if ticket.epoch != s.inner.epoch
            || ticket.volume != s.inner.volume
            || ticket.region != self.state.id
        {
            return Err(not_written(NotWrittenCause::ForeignPosition, c));
        }
        if ticket.generation != self.state.generation {
            return Err(not_written(NotWrittenCause::StaleGeneration, c));
        }
        // Already durable: no barrier. A forced region on a power-safe store
        // counts only what a data flush made durable (D-102).
        let flushes = s.inner.policy.flush;
        let synced = flushes || self.state.forced();
        let durable = if synced && !flushes {
            f.synced()
        } else {
            f.durable()
        };
        if durable >= ticket.end {
            s.inner.domain.count_already_durable();
            return Ok(s.receipt(&self.state, 0, durable, 0, true, synced));
        }
        let domain = &s.inner.domain;
        if !f.wait_completed(ticket.end, || domain.poisoned().is_some()) {
            return Err(Error::Poisoned {
                first: domain.poisoned().unwrap_or(FirstCause {
                    op: None,
                    raw: None,
                }),
            });
        }
        // Snapshot the completed prefix BEFORE the barrier takes its ticket.
        let through = f.completed();
        let (need, synced) = s.barrier_for(&self.state)?;
        if synced {
            f.publish_synced(through);
        } else {
            f.publish_durable(through);
        }
        Ok(s.receipt(&self.state, 0, through, need, true, synced))
    }

    /// Appends `data` and makes it durable before returning.
    ///
    /// # Errors
    ///
    /// As [`Self::append`] and [`Self::sync_through`].
    pub fn append_durable(&self, data: &[u8]) -> Result<(RegionPos, DurableReceipt), Error> {
        let t = self.append(data)?;
        let pos = t.pos();
        let r = self.sync_through(&t)?;
        Ok((pos, r))
    }

    /// A batch: many records packed back to back, written together and made
    /// durable with one barrier. See [`AppendBatch`].
    #[must_use]
    pub fn batch(&self) -> AppendBatch<P> {
        AppendBatch::new(self.clone())
    }

    /// Reads raw bytes at a block-aligned offset into `out` (any length).
    /// Returns the bytes read, fewer than `out.len()` only at the end of the
    /// region's data area.
    ///
    /// The bytes are read into store-io's own aligned buffers and copied
    /// into `out`: direct I/O needs aligned memory, and a read whose wait
    /// fails stays in flight, so it must never target the caller's memory.
    /// To read with no copy, use [`Self::scan`] (it lends store-io's
    /// buffers) or a [`crate::Submitter`]'s `read_append` (it reads into a
    /// buffer taken from the store).
    ///
    /// # Errors
    ///
    /// `NotWritten` for a misaligned or out-of-range read;
    /// `Corruption(MediaError)` if the device reports an unreadable range,
    /// with the exact range in the error's context; `Io` with the raw code
    /// for any other failure.
    pub fn read(&self, offset: u64, out: &mut [u8]) -> Result<usize, Error> {
        let _pass = self.pass()?;
        self.store.read_at(&self.state, offset, out)
    }

    /// Reads the region in order from the block-aligned offset `from` to the
    /// end of its data area, handing every piece to `visit`; return
    /// [`ControlFlow::Break`] to stop (typically where your valid data ends).
    ///
    /// The scan reads with direct I/O and several large reads in flight. It
    /// never stops at bad data: a range the device cannot read is narrowed
    /// to whole sectors (the device's logical blocks where the probe
    /// reported the direct-I/O alignment, else store blocks), reported as
    /// [`ScanItem::Unreadable`], and skipped.
    /// Scans work on read-only and poisoned stores.
    ///
    /// `visit` may call the store (read, write, provision): it never runs
    /// while the scan holds a queue another call waits for (on a shared
    /// queue the scan puts it back before delivering). This region's
    /// `recycle` and `release` wait for the scan itself to end, so they
    /// must not be called from `visit`; nor may an append wait there for
    /// room that only the same thread's [`crate::Submitter`] can free.
    ///
    /// # Errors
    ///
    /// `NotWritten(Misaligned | OutOfBounds | PoolExhausted)` before any I/O;
    /// `Io` if a read fails for a reason other than the media.
    pub fn scan(
        &self,
        from: u64,
        visit: impl FnMut(ScanItem<'_>) -> ControlFlow<()>,
    ) -> Result<ScanSummary, Error> {
        let _pass = self.pass()?;
        self.store
            .scan_reported(&self.state, from, self.state.map.len(), visit)
    }

    /// Moves the append position forward to `offset` (rounded up to a block),
    /// typically after recovery found where valid data ends. Never moves
    /// backwards, and only while no append is in flight.
    ///
    /// The skipped range counts as durable: a writable open flushed the
    /// device before reading anything, so whatever recovery found is on
    /// media. Bytes appended in this open and not yet made durable never do:
    /// then the durable point stays where it is, and the next sync that
    /// flushes covers both (`durable_through` reads lower until then).
    /// A read-only open flushes nothing, so there `resume_at` is refused
    /// (D-072); appends are refused there anyway, and so is a poisoned
    /// store.
    ///
    /// # Errors
    ///
    /// `NotWritten(ReadOnly | TooManyInFlight | OutOfBounds)`, `Poisoned`.
    pub fn resume_at(&self, offset: u64) -> Result<u64, Error> {
        if self.store.inner.read_only || self.state.is_read_only() {
            return Err(Error::NotWritten {
                cause: NotWrittenCause::ReadOnly,
                ctx: ctx(&self.state),
            });
        }
        // A poison landing after this check is harmless: a failed append
        // leaves the prefix below the tail (refused below), a failed flush
        // leaves the durable point below it (not raised).
        if let Some(first) = self.store.inner.domain.poisoned() {
            return Err(Error::Poisoned { first });
        }
        let _pass = self.pass()?;
        let at = self
            .frontier()?
            .resume_at(offset)
            .map_err(|cause| Error::NotWritten {
                cause,
                ctx: ctx(&self.state),
            })?;
        self.state
            .positioned
            .store(true, std::sync::atomic::Ordering::Release);
        Ok(at)
    }

    /// Where the next append will start.
    #[must_use]
    pub fn tail(&self) -> u64 {
        self.state
            .frontier
            .as_ref()
            .map_or(0, crate::frontier::AppendFrontier::tail)
    }

    /// Every byte before this offset is durable.
    #[must_use]
    pub fn durable_through(&self) -> u64 {
        self.state
            .frontier
            .as_ref()
            .map_or(0, crate::frontier::AppendFrontier::durable)
    }

    /// Data-area size in bytes.
    #[must_use]
    pub fn len(&self) -> u64 {
        self.state.map.len()
    }

    /// How far the data area can grow without new space: the region's
    /// capacity in bytes (the sum of its extents).
    #[must_use]
    pub fn capacity(&self) -> u64 {
        self.state.map.capacity()
    }

    /// Grows the data area to at least `new_size` bytes (rounded up to the
    /// block size): the new range is filled, flushed and verified before
    /// one table page write commits the new size (spec ยง12). Within the
    /// capacity nothing else moves; past it the region gains one extent of
    /// at least 1 MiB and an eighth of its capacity (after its last one
    /// when that space is free), listed in its header before the commit.
    /// Returns the new size; on a writable store a size at most the current
    /// one writes nothing and returns it. Positions and tickets taken
    /// before stay valid.
    ///
    /// # Errors
    ///
    /// `Unsupported` in container format 1; `NotWritten` (`TooLarge` past
    /// 2^32 blocks; `ReadOnly`; `Frozen` while a table page is stale or
    /// lost; `NotReady` if the filled range does not verify;
    /// `StaleGeneration` for a recycled region; `QueueFull`); `NoSpace`
    /// (`RegionLimit` for a capped region, `Extents` when its header lists
    /// as many extents as it can hold, `Caller` past the tag's cap,
    /// `Growth` when the device runs out); `Io` if the range could not be
    /// probed or allocated; `Corruption` for metadata that does not decode;
    /// `Poisoned`; `DurabilityUnknown` (a fill, a flush, the header or the
    /// page write failed after submission: poisoned).
    pub fn grow(&self, new_size: u64) -> Result<u64, Error> {
        let _pass = self.pass()?;
        self.store.grow_region2(&self.state, new_size)
    }

    /// Whether the data area is empty (never for a provisioned region).
    #[must_use]
    pub fn is_empty(&self) -> bool {
        self.state.map.len() == 0
    }

    /// Block size: appends are padded to it and reads must be aligned to it.
    #[must_use]
    pub fn block_size(&self) -> u64 {
        self.store.inner.layout.block()
    }
}

// ---------------------------------------------------------------------------
// Page regions
// ---------------------------------------------------------------------------

/// A region of caller-addressed blocks, overwritten in place.
pub struct PageRegion<P: Platform> {
    store: Store<P>,
    state: Arc<LiveRegion>,
}

impl<P: Platform> PageRegion<P> {
    pub(crate) fn from_parts(store: Store<P>, state: Arc<LiveRegion>) -> Self {
        Self { store, state }
    }
}

impl<P: Platform> Clone for PageRegion<P> {
    fn clone(&self) -> Self {
        Self {
            store: self.store.clone(),
            state: Arc::clone(&self.state),
        }
    }
}

impl<P: Platform> PageRegion<P>
where
    P::Queue: Send,
{
    /// A checked position: `offset` must be block-aligned and inside the
    /// region.
    ///
    /// # Errors
    ///
    /// `NotWritten(Misaligned | OutOfBounds)`.
    pub fn pos(&self, offset: u64) -> Result<RegionPos, Error> {
        let c = ctx(&self.state);
        if offset % self.store.block() != 0 {
            return Err(not_written(NotWrittenCause::Misaligned, c));
        }
        if offset >= self.state.map.len() {
            return Err(not_written(NotWrittenCause::OutOfBounds, c));
        }
        Ok(RegionPos {
            volume: self.store.inner.volume,
            region: self.state.id,
            generation: self.state.generation,
            offset,
        })
    }

    /// Checks a write of `len` bytes at `pos`; returns its end.
    pub(crate) fn check(&self, pos: &RegionPos, len: usize) -> Result<u64, Error> {
        let c = ctx(&self.state);
        if pos.volume != self.store.inner.volume || pos.region != self.state.id {
            return Err(not_written(NotWrittenCause::ForeignPosition, c));
        }
        if pos.generation != self.state.generation {
            return Err(not_written(NotWrittenCause::StaleGeneration, c));
        }
        if len == 0 {
            return Err(not_written(NotWrittenCause::Empty, c));
        }
        if (len as u64) % self.store.block() != 0 {
            return Err(not_written(NotWrittenCause::Misaligned, c));
        }
        match pos.offset.checked_add(len as u64) {
            Some(end) if end <= self.state.map.len() => Ok(end),
            _ => Err(not_written(NotWrittenCause::OutOfBounds, c)),
        }
    }

    pub(crate) fn store(&self) -> &Store<P> {
        &self.store
    }

    pub(crate) fn state(&self) -> &Arc<LiveRegion> {
        &self.state
    }

    pub(crate) fn pass(&self) -> Result<Pass<'_>, Error> {
        self.state
            .gate
            .enter()
            .map_err(|why| retired_error(why, &self.state))
    }

    /// The generation this handle writes and reads.
    #[must_use]
    pub fn generation(&self) -> Generation {
        self.state.generation
    }

    /// Starts a new generation (see [`AppendRegion::recycle`]); positions and
    /// tickets of the old generation are refused from then on.
    ///
    /// # Errors
    ///
    /// As [`AppendRegion::recycle`].
    pub fn recycle(&self) -> Result<Self, Error> {
        let state = self.store.recycle_region(&self.state)?;
        Ok(Self {
            store: self.store.clone(),
            state,
        })
    }

    /// Releases the region (see [`AppendRegion::release`]).
    ///
    /// # Errors
    ///
    /// As [`AppendRegion::release`].
    pub fn release(self) -> Result<(), Error> {
        self.store.release_region(&self.state)
    }

    /// Writes whole blocks at `pos` and returns a ticket once the write has
    /// completed. `data.len()` must be a multiple of [`Self::block_size`];
    /// any size is accepted (large writes go out in several pieces).
    ///
    /// # Errors
    ///
    /// `NotWritten` (foreign, stale, misaligned, out of range, pool
    /// exhausted), `Io` (refused by the platform before reaching the device;
    /// nothing is poisoned), `Poisoned`, or `DurabilityUnknown`.
    pub fn write(&self, pos: RegionPos, data: &[u8]) -> Result<WriteTicket, Error> {
        let s = &self.store;
        let _pass = self.pass()?;
        s.ready_for(&self.state)?;
        let end = self.check(&pos, data.len())?;
        let c = ctx_range(&self.state, pos.offset, end);
        let outcome = s.write_staged(&self.state, pos.offset, data, c)?;
        let wrote = Wrote::Page {
            start: pos.offset,
            end,
        };
        settle_pumped(s, &self.state, wrote, outcome)
    }

    /// Makes the ticket's write durable, together with every other write to
    /// this store that completed before the call.
    ///
    /// # Errors
    ///
    /// `NotWritten(ForeignPosition | StaleGeneration)`, `Poisoned`, or
    /// `DurabilityUnknown`.
    pub fn sync_through(&self, ticket: &WriteTicket) -> Result<DurableReceipt, Error> {
        let s = &self.store;
        let c = ctx(&self.state);
        if ticket.epoch != s.inner.epoch
            || ticket.volume != s.inner.volume
            || ticket.region != self.state.id
        {
            return Err(not_written(NotWrittenCause::ForeignPosition, c));
        }
        if ticket.generation != self.state.generation {
            return Err(not_written(NotWrittenCause::StaleGeneration, c));
        }
        self.state
            .gate
            .check()
            .map_err(|why| retired_error(why, &self.state))?;
        let (need, synced) = s.barrier_for(&self.state)?;
        Ok(s.receipt(&self.state, ticket.start, ticket.end, need, false, synced))
    }

    /// Writes whole blocks at `pos` and makes them durable before returning.
    ///
    /// # Errors
    ///
    /// As [`Self::write`] and [`Self::sync_through`].
    pub fn write_durable(&self, pos: RegionPos, data: &[u8]) -> Result<DurableReceipt, Error> {
        let t = self.write(pos, data)?;
        self.sync_through(&t)
    }

    /// A batch: many page writes submitted together and made durable with
    /// one barrier. See [`PageBatch`].
    #[must_use]
    pub fn batch(&self) -> PageBatch<P> {
        PageBatch::new(self.clone())
    }

    /// Reads at `pos` into `out` (any length; reads whole blocks internally).
    /// Returns the bytes read, fewer than `out.len()` only at the end of the
    /// region.
    ///
    /// The bytes are read into store-io's own aligned buffers and copied
    /// into `out`: direct I/O needs aligned memory, and a read whose wait
    /// fails stays in flight, so it must never target the caller's memory.
    /// To read with no copy, use [`Self::scan`] (it lends store-io's
    /// buffers) or a [`crate::Submitter`]'s `read_page` (it reads into a
    /// buffer taken from the store).
    ///
    /// # Errors
    ///
    /// `NotWritten` for a foreign, stale or out-of-range position;
    /// `Corruption(MediaError)` for an unreadable range, with the exact range
    /// in the error's context; `Io` with the raw code for any other failure.
    pub fn read(&self, pos: RegionPos, out: &mut [u8]) -> Result<usize, Error> {
        let c = ctx(&self.state);
        if pos.volume != self.store.inner.volume || pos.region != self.state.id {
            return Err(not_written(NotWrittenCause::ForeignPosition, c));
        }
        if pos.generation != self.state.generation {
            return Err(not_written(NotWrittenCause::StaleGeneration, c));
        }
        let _pass = self.pass()?;
        self.store.read_at(&self.state, pos.offset, out)
    }

    /// Reads the region in order from the block-aligned offset `from` to the
    /// end of its data area, handing every piece to `visit`; return
    /// [`ControlFlow::Break`] to stop (typically where your valid data ends).
    ///
    /// The scan reads with direct I/O and several large reads in flight. It
    /// never stops at bad data: a range the device cannot read is narrowed
    /// to whole sectors (the device's logical blocks where the probe
    /// reported the direct-I/O alignment, else store blocks), reported as
    /// [`ScanItem::Unreadable`], and skipped.
    /// Scans work on read-only and poisoned stores.
    ///
    /// `visit` may call the store (read, write, provision): it never runs
    /// while the scan holds a queue another call waits for (on a shared
    /// queue the scan puts it back before delivering). This region's
    /// `recycle` and `release` wait for the scan itself to end, so they
    /// must not be called from `visit`; nor may an append wait there for
    /// room that only the same thread's [`crate::Submitter`] can free.
    ///
    /// # Errors
    ///
    /// `NotWritten(Misaligned | OutOfBounds | PoolExhausted)` before any I/O;
    /// `Io` if a read fails for a reason other than the media.
    pub fn scan(
        &self,
        from: u64,
        visit: impl FnMut(ScanItem<'_>) -> ControlFlow<()>,
    ) -> Result<ScanSummary, Error> {
        let _pass = self.pass()?;
        self.store
            .scan_reported(&self.state, from, self.state.map.len(), visit)
    }

    /// Block size: writes must be whole blocks.
    #[must_use]
    pub fn block_size(&self) -> u64 {
        self.store.inner.layout.block()
    }

    /// Data-area size in bytes.
    #[must_use]
    pub fn len(&self) -> u64 {
        self.state.map.len()
    }

    /// How far the data area can grow without new space: the region's
    /// capacity in bytes (the sum of its extents).
    #[must_use]
    pub fn capacity(&self) -> u64 {
        self.state.map.capacity()
    }

    /// Grows the data area to at least `new_size` bytes (rounded up to the
    /// block size): the new range is filled, flushed and verified before
    /// one table page write commits the new size (spec ยง12). Within the
    /// capacity nothing else moves; past it the region gains one extent of
    /// at least 1 MiB and an eighth of its capacity (after its last one
    /// when that space is free), listed in its header before the commit.
    /// Returns the new size; on a writable store a size at most the current
    /// one writes nothing and returns it. Positions and tickets taken
    /// before stay valid.
    ///
    /// # Errors
    ///
    /// `Unsupported` in container format 1; `NotWritten` (`TooLarge` past
    /// 2^32 blocks; `ReadOnly`; `Frozen` while a table page is stale or
    /// lost; `NotReady` if the filled range does not verify;
    /// `StaleGeneration` for a recycled region; `QueueFull`); `NoSpace`
    /// (`RegionLimit` for a capped region, `Extents` when its header lists
    /// as many extents as it can hold, `Caller` past the tag's cap,
    /// `Growth` when the device runs out); `Io` if the range could not be
    /// probed or allocated; `Corruption` for metadata that does not decode;
    /// `Poisoned`; `DurabilityUnknown` (a fill, a flush, the header or the
    /// page write failed after submission: poisoned).
    pub fn grow(&self, new_size: u64) -> Result<u64, Error> {
        let _pass = self.pass()?;
        self.store.grow_region2(&self.state, new_size)
    }

    /// Whether the data area is empty (never for a provisioned region).
    #[must_use]
    pub fn is_empty(&self) -> bool {
        self.state.map.len() == 0
    }
}

impl core::fmt::Debug for LiveRegion {
    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
        f.debug_struct("Region")
            .field("name", &self.name.as_str())
            .field("id", &self.id)
            .field("kind", &self.kind)
            .field("generation", &self.generation)
            .field("size", &self.map.len())
            .finish()
    }
}

impl<P: Platform> core::fmt::Debug for AppendRegion<P> {
    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
        f.debug_tuple("AppendRegion").field(&*self.state).finish()
    }
}

impl<P: Platform> core::fmt::Debug for PageRegion<P> {
    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
        f.debug_tuple("PageRegion").field(&*self.state).finish()
    }
}