Skip to main content

store_io_engine/
region.rs

1//! Region handles: [`AppendRegion`], [`PageRegion`] and [`Slot`].
2//!
3//! Every handle is cheap to clone and safe to share between threads. Calls
4//! block until their I/O is done.
5
6use std::ops::ControlFlow;
7use std::sync::Arc;
8use std::sync::atomic::Ordering;
9
10use store_io_core::error::{Error, FirstCause, Named, NotWrittenCause};
11use store_io_core::id::Generation;
12use store_io_format::meta::RegionKind;
13use store_io_platform::Platform;
14
15use crate::batch::{AppendBatch, PageBatch};
16use crate::gate::Pass;
17use crate::io::{ctx, ctx_range, not_written};
18use crate::lifecycle::retired_error;
19use crate::mapped::Mapped;
20use crate::options::RegionOptions;
21use crate::receipt::{DurableReceipt, RegionPos, WriteTicket};
22use crate::scan::{ScanItem, ScanSummary};
23use crate::settle::{Wrote, settle_pumped};
24use crate::store::{LiveRegion, Store};
25
26pub use crate::slot_handle::Slot;
27
28/// Region lookups and provisioning.
29impl<P: Platform> Store<P>
30where
31    P::Queue: Send,
32{
33    /// Looks up an existing append region.
34    ///
35    /// # Errors
36    ///
37    /// [`Error::NotFound`] if no ready append region has that name.
38    pub fn append_region(&self, name: &str) -> Result<AppendRegion<P>, Error> {
39        let state = self.find(name, RegionKind::Append, Named::Region)?;
40        Ok(AppendRegion {
41            store: self.clone(),
42            state,
43        })
44    }
45
46    /// Provisions a new append region of at least `size` bytes.
47    ///
48    /// # Errors
49    ///
50    /// [`Error::AlreadyExists`], [`Error::NoSpace`], or a platform error.
51    pub fn provision_append_region(&self, name: &str, size: u64) -> Result<AppendRegion<P>, Error> {
52        self.provision_append_region_with(name, RegionOptions::default().size(size))
53    }
54
55    /// Provisions a new append region as `opts` say (its size, and in
56    /// container format 2 its cap).
57    ///
58    /// # Errors
59    ///
60    /// As [`Self::provision_append_region`]; `NotWritten(InvalidOptions)`
61    /// for options [`RegionOptions::validate`] refuses; `Unsupported` for a
62    /// cap in container format 1.
63    pub fn provision_append_region_with(
64        &self,
65        name: &str,
66        opts: RegionOptions,
67    ) -> Result<AppendRegion<P>, Error> {
68        let state = self.provision(name, RegionKind::Append, opts, None)?;
69        Ok(AppendRegion {
70            store: self.clone(),
71            state,
72        })
73    }
74
75    /// Provisions a new append region with the store's
76    /// [`StoreOptions::region_defaults`](crate::StoreOptions::region_defaults).
77    ///
78    /// ```
79    /// # use std::path::Path;
80    /// # use store_io_engine::{RegionOptions, Store, StoreOptions};
81    /// # use store_io_sim::{SimConfig, SimPlatform};
82    /// let opts = StoreOptions {
83    ///     region_defaults: Some(RegionOptions::default().size(1 << 20)),
84    ///     ..StoreOptions::default()
85    /// };
86    /// let store = Store::create(SimPlatform::new(SimConfig::volatile(1)), Path::new("/db"), opts)?;
87    /// let log = store.provision_append_region_default("log")?;
88    /// assert_eq!(log.len(), 1 << 20);
89    /// # Ok::<(), store_io_core::error::Error>(())
90    /// ```
91    ///
92    /// # Errors
93    ///
94    /// As [`Self::provision_append_region_with`];
95    /// `NotWritten(InvalidOptions)` when the store has no defaults (before
96    /// any other refusal).
97    pub fn provision_append_region_default(&self, name: &str) -> Result<AppendRegion<P>, Error> {
98        self.provision_append_region_with(name, self.region_defaults()?)
99    }
100
101    /// Provisions a new page region with the store's
102    /// [`StoreOptions::region_defaults`](crate::StoreOptions::region_defaults).
103    ///
104    /// # Errors
105    ///
106    /// As [`Self::provision_append_region_default`].
107    pub fn provision_page_region_default(&self, name: &str) -> Result<PageRegion<P>, Error> {
108        self.provision_page_region_with(name, self.region_defaults()?)
109    }
110
111    /// The store's region defaults; refused when it has none (before any
112    /// I/O, and before the store's own refusals: read-only, poisoned).
113    pub(crate) fn region_defaults(&self) -> Result<RegionOptions, Error> {
114        self.inner.opts.region_defaults.ok_or(Error::NotWritten {
115            cause: NotWrittenCause::InvalidOptions,
116            ctx: store_io_core::error::ErrorContext::default(),
117        })
118    }
119
120    /// Looks up an existing page region.
121    ///
122    /// # Errors
123    ///
124    /// [`Error::NotFound`] if no ready page region has that name.
125    pub fn page_region(&self, name: &str) -> Result<PageRegion<P>, Error> {
126        let state = self.find(name, RegionKind::Page, Named::Region)?;
127        Ok(PageRegion {
128            store: self.clone(),
129            state,
130        })
131    }
132
133    /// Provisions a new page region of at least `size` bytes.
134    ///
135    /// # Errors
136    ///
137    /// [`Error::AlreadyExists`], [`Error::NoSpace`], or a platform error.
138    pub fn provision_page_region(&self, name: &str, size: u64) -> Result<PageRegion<P>, Error> {
139        self.provision_page_region_with(name, RegionOptions::default().size(size))
140    }
141
142    /// Provisions a new page region as `opts` say (its size, and in
143    /// container format 2 its cap).
144    ///
145    /// # Errors
146    ///
147    /// As [`Self::provision_append_region_with`].
148    pub fn provision_page_region_with(
149        &self,
150        name: &str,
151        opts: RegionOptions,
152    ) -> Result<PageRegion<P>, Error> {
153        let state = self.provision(name, RegionKind::Page, opts, None)?;
154        Ok(PageRegion {
155            store: self.clone(),
156            state,
157        })
158    }
159
160    /// Looks up an existing small-object slot.
161    ///
162    /// # Errors
163    ///
164    /// [`Error::NotFound`] if no slot has that name.
165    pub fn slot(&self, name: &str) -> Result<Slot<P>, Error> {
166        let state = self.find(name, RegionKind::Slot, Named::Slot)?;
167        Ok(Slot::from_parts(self.clone(), state))
168    }
169
170    /// Provisions a new small-object slot. Its capacity is the block size
171    /// minus 176 bytes of headers ([`Slot::capacity`]).
172    ///
173    /// # Errors
174    ///
175    /// [`Error::AlreadyExists`], [`Error::NoSpace`], or a platform error.
176    pub fn provision_slot(&self, name: &str) -> Result<Slot<P>, Error> {
177        let state = self.provision(name, RegionKind::Slot, RegionOptions::default(), None)?;
178        Ok(Slot::from_parts(self.clone(), state))
179    }
180}
181
182// ---------------------------------------------------------------------------
183// Append regions
184// ---------------------------------------------------------------------------
185
186/// A region store-io appends to. Positions are chosen by store-io from an
187/// atomic tail, so appends from many threads never overlap and a written
188/// block is never rewritten.
189///
190/// Each append occupies whole blocks: the bytes from the end of the data to
191/// the next block boundary are zero and are not caller data. Pack small
192/// records into one append (or use a batch) to avoid the padding.
193pub struct AppendRegion<P: Platform> {
194    store: Store<P>,
195    state: Arc<LiveRegion>,
196}
197
198impl<P: Platform> AppendRegion<P> {
199    pub(crate) fn from_parts(store: Store<P>, state: Arc<LiveRegion>) -> Self {
200        Self { store, state }
201    }
202}
203
204impl<P: Platform> Clone for AppendRegion<P> {
205    fn clone(&self) -> Self {
206        Self {
207            store: self.store.clone(),
208            state: Arc::clone(&self.state),
209        }
210    }
211}
212
213/// The tail of every prefix sync (`sync_through`, a set barrier's append
214/// tickets, `settle::durable_receipt`): publishes `snapshot`, the completed
215/// prefix read **before** the flush's ticket `need` was taken (a prefix read
216/// after it may hold bytes the flush does not cover), as synced when a data
217/// flush backs it, else durable, and builds its prefix receipt.
218pub(crate) fn sync_tail<P: Platform>(
219    store: &Store<P>,
220    state: &LiveRegion,
221    snapshot: u64,
222    need: u64,
223    synced: bool,
224) -> DurableReceipt
225where
226    P::Queue: Send,
227{
228    if let Some(f) = state.frontier.as_ref() {
229        if synced {
230            f.publish_synced(snapshot);
231        } else {
232            f.publish_durable(snapshot);
233        }
234    }
235    store.receipt(state, 0, snapshot, need, true, synced)
236}
237impl<P: Platform> AppendRegion<P>
238where
239    P::Queue: Send,
240{
241    /// Threads waiting on the region's completed prefix (an append waiting
242    /// for room in the window, a sync waiting for its appends). For tests.
243    #[doc(hidden)]
244    #[must_use]
245    pub fn waiting_on_appends(&self) -> u64 {
246        self.state.frontier.as_ref().map_or(0, |f| f.waiting())
247    }
248
249    fn frontier(&self) -> Result<&crate::frontier::AppendFrontier, Error> {
250        self.state
251            .frontier
252            .as_ref()
253            .ok_or_else(|| not_written(NotWrittenCause::NotReady, ctx(&self.state)))
254    }
255
256    /// The checks every append makes before touching the device, and the
257    /// pass that keeps a recycle or release out until the append is done.
258    pub(crate) fn writable(&self) -> Result<(&crate::frontier::AppendFrontier, Pass<'_>), Error> {
259        let pass = self.pass()?;
260        self.store.ready_for(&self.state)?;
261        let f = self.frontier()?;
262        if !self.state.positioned.load(Ordering::Acquire) {
263            return Err(not_written(
264                NotWrittenCause::NotPositioned,
265                ctx(&self.state),
266            ));
267        }
268        Ok((f, pass))
269    }
270
271    fn pass(&self) -> Result<Pass<'_>, Error> {
272        self.state
273            .gate
274            .enter()
275            .map_err(|why| retired_error(why, &self.state))
276    }
277
278    /// The generation this handle writes and reads. Recycling starts the
279    /// next one; embed it in your records to tell generations apart after a
280    /// crash.
281    #[must_use]
282    pub fn generation(&self) -> Generation {
283        self.state.generation
284    }
285
286    /// Starts a new generation: waits for every operation in flight on this
287    /// region, rewrites the region header with the next generation, makes it
288    /// durable, and returns a handle to the new generation with its append
289    /// position at 0. No other I/O: the old bytes stay on disk, so records
290    /// must carry their generation. Every handle, ticket and position of the
291    /// old generation is refused from then on (`StaleGeneration`).
292    ///
293    /// Must not be called from inside this region's own scan visitor, nor by
294    /// a thread whose own [`crate::Submitter`] has operations on this region
295    /// it has not reaped: recycle waits for them, and only that thread's
296    /// `reap` or `wait` finishes them (the same holds for `release`).
297    ///
298    /// # Errors
299    ///
300    /// `NotWritten(StaleGeneration | NotReady)` if the region was already
301    /// recycled or released; `NotWritten(ReadOnly)` if the store or the
302    /// region is read-only; `Io` if the store's queue could not be prepared
303    /// (nothing was written; the region stays usable); `Poisoned`;
304    /// `DurabilityUnknown`.
305    pub fn recycle(&self) -> Result<Self, Error> {
306        let state = self.store.recycle_region(&self.state)?;
307        Ok(Self {
308            store: self.store.clone(),
309            state,
310        })
311    }
312
313    /// Releases the region: waits for every operation in flight, records
314    /// the release in the region table (durable: from then on the release
315    /// stands), marks its header released (durable), and gives its space
316    /// back to the file system. Its contents are undefined from then on;
317    /// every handle to it is refused (`NotReady`), and its space is reused
318    /// by later provisioning. A header write that fails before it reaches
319    /// the device (refused, a queue that could not be prepared, a header
320    /// pair that would not encode or plan) still returns success: the space
321    /// is free for provisioning, but not given back to the file system. Not
322    /// from a thread with unreaped [`crate::Submitter`] operations on it
323    /// (see [`Self::recycle`]).
324    ///
325    /// # Errors
326    ///
327    /// `NotWritten(StaleGeneration | NotReady)` if the region was already
328    /// recycled or released; `NotWritten(ReadOnly)` if the store or the
329    /// region is read-only; on format 2, `NotWritten(Frozen)` while a table
330    /// page is stale or lost, and `NotWritten(Overlap)` when removing the
331    /// entry would change another region's verdict at the next open;
332    /// `Poisoned`; `DurabilityUnknown`; or `Io` if the space could not be
333    /// deallocated (the region is released either way). Once the table
334    /// records the release, a release fails only on a poisoned store or
335    /// with that `Io`.
336    pub fn release(self) -> Result<(), Error> {
337        self.store.release_region(&self.state)
338    }
339
340    pub(crate) fn store(&self) -> &Store<P> {
341        &self.store
342    }
343
344    pub(crate) fn state(&self) -> &Arc<LiveRegion> {
345        &self.state
346    }
347
348    /// Appends `data` (any length of at least one byte) and returns a ticket
349    /// once the write has completed. The data is not yet durable: pass the
350    /// ticket to [`Self::sync_through`], or use [`Self::append_durable`].
351    ///
352    /// The append starts at a block boundary chosen by store-io and is
353    /// zero-padded to the next one. Data larger than the largest pooled
354    /// buffer is written in several pieces, kept in flight together.
355    ///
356    /// # Errors
357    ///
358    /// `NotWritten` (empty data, region full, pool exhausted, read-only, not
359    /// positioned), `Io` if the store's queue could not be prepared (nothing
360    /// was reserved or written), `Poisoned`, or `DurabilityUnknown` (the
361    /// domain is now poisoned).
362    pub fn append(&self, data: &[u8]) -> Result<WriteTicket, Error> {
363        self.append_with(data, || Ok(()), |()| {})
364    }
365
366    /// [`Self::append`], running `before_reserve` once its lane is held and
367    /// before its reservation (again on each retry for room), and handing
368    /// what it returned to `on_reserved` once the space is reserved, before
369    /// the write is submitted (a dependent write's gate passes, `after.rs`).
370    pub(crate) fn append_with<T>(
371        &self,
372        data: &[u8],
373        before_reserve: impl FnMut() -> Result<T, Error>,
374        on_reserved: impl FnOnce(T),
375    ) -> Result<WriteTicket, Error> {
376        let s = &self.store;
377        let (f, _pass) = self.writable()?;
378        let c = ctx(&self.state);
379        if data.is_empty() {
380            return Err(not_written(NotWrittenCause::Empty, c));
381        }
382        // Everything that can fail without touching the device happens first:
383        // the first piece is staged before any space is reserved.
384        let mut stream = s.stage(data, c)?;
385        let (mut lane, reserved, before) =
386            s.lane_and_reservation_with(&self.state, f, data.len() as u64, c, before_reserve)?;
387        on_reserved(before);
388        let r = reserved.range();
389        stream.at(r.start);
390        // Until the pump returns the guard poisons on any exit (a panic
391        // included); then the settle step completes the reservation, or
392        // poisons: past it every failure leaves a hole.
393        let outcome = s.pump(
394            &self.state,
395            &mut lane,
396            &mut Mapped::new(&self.state.map, &mut stream),
397        );
398        let r = reserved.disarm();
399        settle_pumped(s, &self.state, Wrote::Append(r), outcome)
400    }
401
402    /// Makes everything appended up to and including `ticket` durable.
403    ///
404    /// # Errors
405    ///
406    /// `NotWritten(ForeignPosition | StaleGeneration)` for a ticket of another
407    /// region or generation; `Poisoned`; `DurabilityUnknown` if the flush
408    /// failed.
409    pub fn sync_through(&self, ticket: &WriteTicket) -> Result<DurableReceipt, Error> {
410        self.sync_through_from(ticket, None)
411    }
412
413    /// [`Self::sync_through`] for a commit that entered at `entry`
414    /// (`Store::write_entry`, read before its write).
415    pub(crate) fn sync_through_from(
416        &self,
417        ticket: &WriteTicket,
418        entry: Option<u64>,
419    ) -> Result<DurableReceipt, Error> {
420        let s = &self.store;
421        let f = self.frontier()?;
422        let c = ctx(&self.state);
423        self.state
424            .gate
425            .check()
426            .map_err(|why| retired_error(why, &self.state))?;
427        if ticket.epoch != s.inner.epoch
428            || ticket.volume != s.inner.volume
429            || ticket.region != self.state.id
430        {
431            return Err(not_written(NotWrittenCause::ForeignPosition, c));
432        }
433        if ticket.generation != self.state.generation {
434            return Err(not_written(NotWrittenCause::StaleGeneration, c));
435        }
436        // Already durable: no barrier. A forced region on a power-safe store
437        // counts only what a data flush made durable (D-102).
438        let flushes = s.inner.policy.flush;
439        let synced = flushes || self.state.forced();
440        let durable = if synced && !flushes {
441            f.synced()
442        } else {
443            f.durable()
444        };
445        if durable >= ticket.end {
446            s.inner.domain.count_already_durable();
447            return Ok(s.receipt(&self.state, 0, durable, 0, true, synced));
448        }
449        let domain = &s.inner.domain;
450        if !f.wait_completed(ticket.end, || domain.poisoned().is_some()) {
451            return Err(Error::Poisoned {
452                first: domain.poisoned().unwrap_or(FirstCause {
453                    op: None,
454                    raw: None,
455                }),
456            });
457        }
458        // Snapshot the completed prefix BEFORE the barrier takes its ticket.
459        let through = f.completed();
460        let (need, synced) = s.barrier_for(&self.state, entry)?;
461        Ok(sync_tail(s, &self.state, through, need, synced))
462    }
463
464    /// Appends `data` and makes it durable before returning.
465    ///
466    /// # Errors
467    ///
468    /// As [`Self::append`] and [`Self::sync_through`].
469    pub fn append_durable(&self, data: &[u8]) -> Result<(RegionPos, DurableReceipt), Error> {
470        // Timed for auto-tuning from before the write (D-160).
471        let entry = self.store.write_entry(&self.state);
472        let t = self.append(data)?;
473        let pos = t.pos();
474        let r = self.sync_through_from(&t, entry)?;
475        Ok((pos, r))
476    }
477
478    /// A batch: many records packed back to back, written together and made
479    /// durable with one barrier. See [`AppendBatch`].
480    #[must_use]
481    pub fn batch(&self) -> AppendBatch<P> {
482        AppendBatch::new(self.clone())
483    }
484
485    /// Reads raw bytes at a block-aligned offset into `out` (any length).
486    /// Returns the bytes read, fewer than `out.len()` only at the end of the
487    /// region's data area.
488    ///
489    /// The bytes are read into store-io's own aligned buffers and copied
490    /// into `out`: direct I/O needs aligned memory, and a read whose wait
491    /// fails stays in flight, so it must never target the caller's memory.
492    /// To read with no copy, use [`Self::scan`] (it lends store-io's
493    /// buffers) or a [`crate::Submitter`]'s `read_append` (it reads into a
494    /// buffer taken from the store).
495    ///
496    /// # Errors
497    ///
498    /// `NotWritten` for a misaligned or out-of-range read;
499    /// `Corruption(MediaError)` if the device reports an unreadable range,
500    /// with the exact range in the error's context; `Io` with the raw code
501    /// for any other failure.
502    pub fn read(&self, offset: u64, out: &mut [u8]) -> Result<usize, Error> {
503        let _pass = self.pass()?;
504        self.store.read_at(&self.state, offset, out)
505    }
506
507    /// Reads the region in order from the block-aligned offset `from` to the
508    /// end of its data area, handing every piece to `visit`; return
509    /// [`ControlFlow::Break`] to stop (typically where your valid data ends).
510    ///
511    /// The scan reads with direct I/O and several large reads in flight. It
512    /// never stops at bad data: a range the device cannot read is narrowed
513    /// to whole sectors (the device's logical blocks where the probe
514    /// reported the direct-I/O alignment, else store blocks), reported as
515    /// [`ScanItem::Unreadable`], and skipped.
516    /// Scans work on read-only and poisoned stores.
517    ///
518    /// `visit` may call the store (read, write, provision): it never runs
519    /// while the scan holds a queue another call waits for (on a shared
520    /// queue the scan puts it back before delivering). This region's
521    /// `recycle` and `release` wait for the scan itself to end, so they
522    /// must not be called from `visit`; nor may an append wait there for
523    /// room that only the same thread's [`crate::Submitter`] can free.
524    ///
525    /// # Errors
526    ///
527    /// `NotWritten(Misaligned | OutOfBounds | PoolExhausted)` before any I/O;
528    /// `Io` if a read fails for a reason other than the media.
529    pub fn scan(
530        &self,
531        from: u64,
532        visit: impl FnMut(ScanItem<'_>) -> ControlFlow<()>,
533    ) -> Result<ScanSummary, Error> {
534        let _pass = self.pass()?;
535        self.store
536            .scan_reported(&self.state, from, self.state.map.len(), visit)
537    }
538
539    /// Moves the append position forward to `offset` (rounded up to a block),
540    /// typically after recovery found where valid data ends. Never moves
541    /// backwards, and only while no append is in flight.
542    ///
543    /// The skipped range counts as durable: a writable open flushed the
544    /// device before reading anything, so whatever recovery found is on
545    /// media. Bytes appended in this open and not yet made durable never do:
546    /// then the durable point stays where it is, and the next sync that
547    /// flushes covers both (`durable_through` reads lower until then).
548    /// A read-only open flushes nothing, so there `resume_at` is refused
549    /// (D-072); appends are refused there anyway, and so is a poisoned
550    /// store.
551    ///
552    /// # Errors
553    ///
554    /// `NotWritten(ReadOnly | TooManyInFlight | OutOfBounds)`, `Poisoned`.
555    pub fn resume_at(&self, offset: u64) -> Result<u64, Error> {
556        if self.store.inner.read_only || self.state.is_read_only() {
557            return Err(Error::NotWritten {
558                cause: NotWrittenCause::ReadOnly,
559                ctx: ctx(&self.state),
560            });
561        }
562        // A poison landing after this check is harmless: a failed append
563        // leaves the prefix below the tail (refused below), a failed flush
564        // leaves the durable point below it (not raised).
565        if let Some(first) = self.store.inner.domain.poisoned() {
566            return Err(Error::Poisoned { first });
567        }
568        let _pass = self.pass()?;
569        let at = self
570            .frontier()?
571            .resume_at(offset)
572            .map_err(|cause| Error::NotWritten {
573                cause,
574                ctx: ctx(&self.state),
575            })?;
576        // Before the region reads as positioned: a dependent write that sees
577        // it positioned sees this (`after.rs`).
578        let inner = &self.store.inner;
579        crate::frontier::record_resume(
580            (inner.volume, self.state.id, self.state.generation),
581            inner.epoch,
582            at,
583        );
584        self.state
585            .positioned
586            .store(true, std::sync::atomic::Ordering::Release);
587        Ok(at)
588    }
589
590    /// Where the next append will start.
591    #[must_use]
592    pub fn tail(&self) -> u64 {
593        self.state
594            .frontier
595            .as_ref()
596            .map_or(0, crate::frontier::AppendFrontier::tail)
597    }
598
599    /// The bytes of `range` (region offsets, any start and end) as a
600    /// [`ReadStream`](crate::ReadStream) of owned chunks in order, with the
601    /// default [`StreamOptions`](crate::StreamOptions): a background
602    /// stream. Only the durable prefix as of the call is read (`range` is
603    /// clamped to [`Self::durable_through`]; empty: an empty stream).
604    ///
605    /// # Errors
606    ///
607    /// `NotWritten`: `OutOfBounds` (`range` reversed or past [`Self::len`]),
608    /// `NotPositioned` (after a reopen, before `resume_at`: so never on a
609    /// read-only open, which refuses `resume_at`; scan it instead),
610    /// `StaleGeneration` or `NotReady` (recycled or released),
611    /// `PoolExhausted` (too few buffers free under the pool's background
612    /// cap), `InvalidOptions`; `Io` if its queue could not be made.
613    pub fn read_stream(&self, range: std::ops::Range<u64>) -> Result<crate::ReadStream<P>, Error> {
614        self.read_stream_with(range, crate::StreamOptions::default())
615    }
616
617    /// [`Self::read_stream`] with `opts`.
618    ///
619    /// # Errors
620    ///
621    /// As [`Self::read_stream`].
622    pub fn read_stream_with(
623        &self,
624        range: std::ops::Range<u64>,
625        opts: crate::StreamOptions,
626    ) -> Result<crate::ReadStream<P>, Error> {
627        let sr = self.stream_range(range)?;
628        self.store.prefetch(&[sr], opts)
629    }
630
631    /// `range` as one range for [`Store::prefetch`](crate::Store::prefetch),
632    /// checked now as [`Self::read_stream`] checks it (and clamped to the
633    /// durable prefix as of this call).
634    ///
635    /// # Errors
636    ///
637    /// As [`Self::read_stream`]: `OutOfBounds`, `NotPositioned`,
638    /// `StaleGeneration`, `NotReady`.
639    pub fn stream_range(
640        &self,
641        range: std::ops::Range<u64>,
642    ) -> Result<crate::StreamRange<'_, P>, Error> {
643        self.state
644            .gate
645            .live()
646            .map_err(|why| retired_error(why, &self.state))?;
647        if !self.state.positioned.load(Ordering::Acquire) {
648            return Err(not_written(
649                NotWrittenCause::NotPositioned,
650                ctx(&self.state),
651            ));
652        }
653        if range.start > range.end || range.end > self.state.map.len() {
654            return Err(not_written(
655                NotWrittenCause::OutOfBounds,
656                crate::io::ctx_range(&self.state, range.start, range.end),
657            ));
658        }
659        // The durable prefix as of the call (R5.5).
660        let end = range.end.min(self.durable_through());
661        let start = range.start.min(end);
662        Ok(crate::StreamRange::new(
663            &self.store,
664            &self.state,
665            start..end,
666        ))
667    }
668
669    /// Every byte before this offset is durable.
670    #[must_use]
671    pub fn durable_through(&self) -> u64 {
672        self.state
673            .frontier
674            .as_ref()
675            .map_or(0, crate::frontier::AppendFrontier::durable)
676    }
677
678    /// Data-area size in bytes.
679    #[must_use]
680    pub fn len(&self) -> u64 {
681        self.state.map.len()
682    }
683
684    /// How far the data area can grow without new space: the region's
685    /// capacity in bytes (the sum of its extents).
686    #[must_use]
687    pub fn capacity(&self) -> u64 {
688        self.state.map.capacity()
689    }
690
691    /// Grows the data area to at least `new_size` bytes (rounded up to the
692    /// block size): the new range is filled, flushed and verified before
693    /// one table page write commits the new size (spec §12). Within the
694    /// capacity nothing else moves; past it the region gains one extent of
695    /// at least 1 MiB and an eighth of its capacity (after its last one
696    /// when that space is free), listed in its header before the commit.
697    /// Returns the new size; on a writable store a size at most the current
698    /// one writes nothing and returns it. Positions and tickets taken
699    /// before stay valid.
700    ///
701    /// # Errors
702    ///
703    /// `Unsupported` in container format 1; `NotWritten` (`TooLarge` past
704    /// 2^32 blocks; `ReadOnly`; `Frozen` while a table page is stale or
705    /// lost; `NotReady` if the filled range does not verify;
706    /// `StaleGeneration` for a recycled region; `QueueFull`); `NoSpace`
707    /// (`RegionLimit` for a capped region, `Extents` when its header lists
708    /// as many extents as it can hold, `Caller` past the tag's cap,
709    /// `Growth` when the device runs out); `Io` if the range could not be
710    /// probed or allocated; `Corruption` for metadata that does not decode;
711    /// `Poisoned`; `DurabilityUnknown` (a fill, a flush, the header or the
712    /// page write failed after submission: poisoned).
713    pub fn grow(&self, new_size: u64) -> Result<u64, Error> {
714        let _pass = self.pass()?;
715        self.store.grow_region2(&self.state, new_size)
716    }
717
718    /// Whether the data area is empty (never for a provisioned region).
719    #[must_use]
720    pub fn is_empty(&self) -> bool {
721        self.state.map.len() == 0
722    }
723
724    /// Block size: appends are padded to it and reads must be aligned to it.
725    #[must_use]
726    pub fn block_size(&self) -> u64 {
727        self.store.inner.layout.block()
728    }
729}
730
731// ---------------------------------------------------------------------------
732// Page regions
733// ---------------------------------------------------------------------------
734
735/// A region of caller-addressed blocks, overwritten in place.
736pub struct PageRegion<P: Platform> {
737    store: Store<P>,
738    state: Arc<LiveRegion>,
739}
740
741impl<P: Platform> PageRegion<P> {
742    pub(crate) fn from_parts(store: Store<P>, state: Arc<LiveRegion>) -> Self {
743        Self { store, state }
744    }
745}
746
747impl<P: Platform> Clone for PageRegion<P> {
748    fn clone(&self) -> Self {
749        Self {
750            store: self.store.clone(),
751            state: Arc::clone(&self.state),
752        }
753    }
754}
755
756impl<P: Platform> PageRegion<P>
757where
758    P::Queue: Send,
759{
760    /// A checked position: `offset` must be block-aligned and inside the
761    /// region.
762    ///
763    /// # Errors
764    ///
765    /// `NotWritten(Misaligned | OutOfBounds)`.
766    pub fn pos(&self, offset: u64) -> Result<RegionPos, Error> {
767        let c = ctx(&self.state);
768        if offset % self.store.block() != 0 {
769            return Err(not_written(NotWrittenCause::Misaligned, c));
770        }
771        if offset >= self.state.map.len() {
772            return Err(not_written(NotWrittenCause::OutOfBounds, c));
773        }
774        Ok(RegionPos {
775            volume: self.store.inner.volume,
776            region: self.state.id,
777            generation: self.state.generation,
778            offset,
779        })
780    }
781
782    /// Checks a write of `len` bytes at `pos`; returns its end.
783    pub(crate) fn check(&self, pos: &RegionPos, len: usize) -> Result<u64, Error> {
784        let c = ctx(&self.state);
785        if pos.volume != self.store.inner.volume || pos.region != self.state.id {
786            return Err(not_written(NotWrittenCause::ForeignPosition, c));
787        }
788        if pos.generation != self.state.generation {
789            return Err(not_written(NotWrittenCause::StaleGeneration, c));
790        }
791        if len == 0 {
792            return Err(not_written(NotWrittenCause::Empty, c));
793        }
794        if (len as u64) % self.store.block() != 0 {
795            return Err(not_written(NotWrittenCause::Misaligned, c));
796        }
797        match pos.offset.checked_add(len as u64) {
798            Some(end) if end <= self.state.map.len() => Ok(end),
799            _ => Err(not_written(NotWrittenCause::OutOfBounds, c)),
800        }
801    }
802
803    pub(crate) fn store(&self) -> &Store<P> {
804        &self.store
805    }
806
807    pub(crate) fn state(&self) -> &Arc<LiveRegion> {
808        &self.state
809    }
810
811    pub(crate) fn pass(&self) -> Result<Pass<'_>, Error> {
812        self.state
813            .gate
814            .enter()
815            .map_err(|why| retired_error(why, &self.state))
816    }
817
818    /// The generation this handle writes and reads.
819    #[must_use]
820    pub fn generation(&self) -> Generation {
821        self.state.generation
822    }
823
824    /// Starts a new generation (see [`AppendRegion::recycle`]); positions and
825    /// tickets of the old generation are refused from then on.
826    ///
827    /// # Errors
828    ///
829    /// As [`AppendRegion::recycle`].
830    pub fn recycle(&self) -> Result<Self, Error> {
831        let state = self.store.recycle_region(&self.state)?;
832        Ok(Self {
833            store: self.store.clone(),
834            state,
835        })
836    }
837
838    /// Releases the region (see [`AppendRegion::release`]).
839    ///
840    /// # Errors
841    ///
842    /// As [`AppendRegion::release`].
843    pub fn release(self) -> Result<(), Error> {
844        self.store.release_region(&self.state)
845    }
846
847    /// Writes whole blocks at `pos` and returns a ticket once the write has
848    /// completed. `data.len()` must be a multiple of [`Self::block_size`];
849    /// any size is accepted (large writes go out in several pieces).
850    ///
851    /// # Errors
852    ///
853    /// `NotWritten` (foreign, stale, misaligned, out of range, pool
854    /// exhausted), `Io` (refused by the platform before reaching the device;
855    /// nothing is poisoned), `Poisoned`, or `DurabilityUnknown`.
856    pub fn write(&self, pos: RegionPos, data: &[u8]) -> Result<WriteTicket, Error> {
857        self.write_with(pos, data, || Ok(()))
858    }
859
860    /// [`Self::write`], running `before_issue` once its lane is held and
861    /// dropping what it returned before the write is issued (a dependent
862    /// write's gate passes, `after.rs`).
863    pub(crate) fn write_with<T>(
864        &self,
865        pos: RegionPos,
866        data: &[u8],
867        before_issue: impl FnMut() -> Result<T, Error>,
868    ) -> Result<WriteTicket, Error> {
869        let s = &self.store;
870        let _pass = self.pass()?;
871        s.ready_for(&self.state)?;
872        let end = self.check(&pos, data.len())?;
873        let c = ctx_range(&self.state, pos.offset, end);
874        let outcome = s.write_staged_with(&self.state, pos.offset, data, c, before_issue)?;
875        let wrote = Wrote::Page {
876            start: pos.offset,
877            end,
878        };
879        settle_pumped(s, &self.state, wrote, outcome)
880    }
881
882    /// Makes the ticket's write durable, together with every other write to
883    /// this store that completed before the call.
884    ///
885    /// # Errors
886    ///
887    /// `NotWritten(ForeignPosition | StaleGeneration)`, `Poisoned`, or
888    /// `DurabilityUnknown`.
889    pub fn sync_through(&self, ticket: &WriteTicket) -> Result<DurableReceipt, Error> {
890        self.sync_through_from(ticket, None)
891    }
892
893    /// [`Self::sync_through`] for a commit that entered at `entry`
894    /// (`Store::write_entry`, read before its write).
895    fn sync_through_from(
896        &self,
897        ticket: &WriteTicket,
898        entry: Option<u64>,
899    ) -> Result<DurableReceipt, Error> {
900        let s = &self.store;
901        let c = ctx(&self.state);
902        if ticket.epoch != s.inner.epoch
903            || ticket.volume != s.inner.volume
904            || ticket.region != self.state.id
905        {
906            return Err(not_written(NotWrittenCause::ForeignPosition, c));
907        }
908        if ticket.generation != self.state.generation {
909            return Err(not_written(NotWrittenCause::StaleGeneration, c));
910        }
911        self.state
912            .gate
913            .check()
914            .map_err(|why| retired_error(why, &self.state))?;
915        let (need, synced) = s.barrier_for(&self.state, entry)?;
916        Ok(s.receipt(&self.state, ticket.start, ticket.end, need, false, synced))
917    }
918
919    /// Writes whole blocks at `pos` and makes them durable before returning.
920    ///
921    /// # Errors
922    ///
923    /// As [`Self::write`] and [`Self::sync_through`].
924    pub fn write_durable(&self, pos: RegionPos, data: &[u8]) -> Result<DurableReceipt, Error> {
925        // Timed for auto-tuning from before the write (D-160).
926        let entry = self.store.write_entry(&self.state);
927        let t = self.write(pos, data)?;
928        self.sync_through_from(&t, entry)
929    }
930
931    /// A batch: many page writes submitted together and made durable with
932    /// one barrier. See [`PageBatch`].
933    #[must_use]
934    pub fn batch(&self) -> PageBatch<P> {
935        PageBatch::new(self.clone())
936    }
937
938    /// Reads at `pos` into `out` (any length; reads whole blocks internally).
939    /// Returns the bytes read, fewer than `out.len()` only at the end of the
940    /// region.
941    ///
942    /// The bytes are read into store-io's own aligned buffers and copied
943    /// into `out`: direct I/O needs aligned memory, and a read whose wait
944    /// fails stays in flight, so it must never target the caller's memory.
945    /// To read with no copy, use [`Self::scan`] (it lends store-io's
946    /// buffers) or a [`crate::Submitter`]'s `read_page` (it reads into a
947    /// buffer taken from the store).
948    ///
949    /// # Errors
950    ///
951    /// `NotWritten` for a foreign, stale or out-of-range position;
952    /// `Corruption(MediaError)` for an unreadable range, with the exact range
953    /// in the error's context; `Io` with the raw code for any other failure.
954    pub fn read(&self, pos: RegionPos, out: &mut [u8]) -> Result<usize, Error> {
955        let c = ctx(&self.state);
956        if pos.volume != self.store.inner.volume || pos.region != self.state.id {
957            return Err(not_written(NotWrittenCause::ForeignPosition, c));
958        }
959        if pos.generation != self.state.generation {
960            return Err(not_written(NotWrittenCause::StaleGeneration, c));
961        }
962        let _pass = self.pass()?;
963        self.store.read_at(&self.state, pos.offset, out)
964    }
965
966    /// Reads the region in order from the block-aligned offset `from` to the
967    /// end of its data area, handing every piece to `visit`; return
968    /// [`ControlFlow::Break`] to stop (typically where your valid data ends).
969    ///
970    /// The scan reads with direct I/O and several large reads in flight. It
971    /// never stops at bad data: a range the device cannot read is narrowed
972    /// to whole sectors (the device's logical blocks where the probe
973    /// reported the direct-I/O alignment, else store blocks), reported as
974    /// [`ScanItem::Unreadable`], and skipped.
975    /// Scans work on read-only and poisoned stores.
976    ///
977    /// `visit` may call the store (read, write, provision): it never runs
978    /// while the scan holds a queue another call waits for (on a shared
979    /// queue the scan puts it back before delivering). This region's
980    /// `recycle` and `release` wait for the scan itself to end, so they
981    /// must not be called from `visit`; nor may an append wait there for
982    /// room that only the same thread's [`crate::Submitter`] can free.
983    ///
984    /// # Errors
985    ///
986    /// `NotWritten(Misaligned | OutOfBounds | PoolExhausted)` before any I/O;
987    /// `Io` if a read fails for a reason other than the media.
988    pub fn scan(
989        &self,
990        from: u64,
991        visit: impl FnMut(ScanItem<'_>) -> ControlFlow<()>,
992    ) -> Result<ScanSummary, Error> {
993        let _pass = self.pass()?;
994        self.store
995            .scan_reported(&self.state, from, self.state.map.len(), visit)
996    }
997
998    /// The bytes of `range` (region offsets, any start and end) as a
999    /// [`ReadStream`](crate::ReadStream) of owned chunks in order, with the
1000    /// default [`StreamOptions`](crate::StreamOptions): a background stream.
1001    /// Not a snapshot: a write made while streaming may show in it.
1002    ///
1003    /// # Errors
1004    ///
1005    /// `NotWritten`: `OutOfBounds` (`range` reversed or past [`Self::len`]),
1006    /// `StaleGeneration` or `NotReady` (recycled or released),
1007    /// `PoolExhausted` (too few buffers free under the pool's background
1008    /// cap), `InvalidOptions`; `Io` if its queue could not be made.
1009    pub fn read_stream(&self, range: std::ops::Range<u64>) -> Result<crate::ReadStream<P>, Error> {
1010        self.read_stream_with(range, crate::StreamOptions::default())
1011    }
1012
1013    /// [`Self::read_stream`] with `opts`.
1014    ///
1015    /// # Errors
1016    ///
1017    /// As [`Self::read_stream`].
1018    pub fn read_stream_with(
1019        &self,
1020        range: std::ops::Range<u64>,
1021        opts: crate::StreamOptions,
1022    ) -> Result<crate::ReadStream<P>, Error> {
1023        let sr = self.stream_range(range)?;
1024        self.store.prefetch(&[sr], opts)
1025    }
1026
1027    /// `range` as one range for [`Store::prefetch`](crate::Store::prefetch),
1028    /// checked now as [`Self::read_stream`] checks it.
1029    ///
1030    /// # Errors
1031    ///
1032    /// As [`Self::read_stream`]: `OutOfBounds`, `StaleGeneration`,
1033    /// `NotReady`.
1034    pub fn stream_range(
1035        &self,
1036        range: std::ops::Range<u64>,
1037    ) -> Result<crate::StreamRange<'_, P>, Error> {
1038        self.state
1039            .gate
1040            .live()
1041            .map_err(|why| retired_error(why, &self.state))?;
1042        if range.start > range.end || range.end > self.state.map.len() {
1043            return Err(not_written(
1044                NotWrittenCause::OutOfBounds,
1045                crate::io::ctx_range(&self.state, range.start, range.end),
1046            ));
1047        }
1048        Ok(crate::StreamRange::new(&self.store, &self.state, range))
1049    }
1050
1051    /// Block size: writes must be whole blocks.
1052    #[must_use]
1053    pub fn block_size(&self) -> u64 {
1054        self.store.inner.layout.block()
1055    }
1056
1057    /// Data-area size in bytes.
1058    #[must_use]
1059    pub fn len(&self) -> u64 {
1060        self.state.map.len()
1061    }
1062
1063    /// How far the data area can grow without new space: the region's
1064    /// capacity in bytes (the sum of its extents).
1065    #[must_use]
1066    pub fn capacity(&self) -> u64 {
1067        self.state.map.capacity()
1068    }
1069
1070    /// Grows the data area to at least `new_size` bytes (rounded up to the
1071    /// block size): the new range is filled, flushed and verified before
1072    /// one table page write commits the new size (spec §12). Within the
1073    /// capacity nothing else moves; past it the region gains one extent of
1074    /// at least 1 MiB and an eighth of its capacity (after its last one
1075    /// when that space is free), listed in its header before the commit.
1076    /// Returns the new size; on a writable store a size at most the current
1077    /// one writes nothing and returns it. Positions and tickets taken
1078    /// before stay valid.
1079    ///
1080    /// # Errors
1081    ///
1082    /// `Unsupported` in container format 1; `NotWritten` (`TooLarge` past
1083    /// 2^32 blocks; `ReadOnly`; `Frozen` while a table page is stale or
1084    /// lost; `NotReady` if the filled range does not verify;
1085    /// `StaleGeneration` for a recycled region; `QueueFull`); `NoSpace`
1086    /// (`RegionLimit` for a capped region, `Extents` when its header lists
1087    /// as many extents as it can hold, `Caller` past the tag's cap,
1088    /// `Growth` when the device runs out); `Io` if the range could not be
1089    /// probed or allocated; `Corruption` for metadata that does not decode;
1090    /// `Poisoned`; `DurabilityUnknown` (a fill, a flush, the header or the
1091    /// page write failed after submission: poisoned).
1092    pub fn grow(&self, new_size: u64) -> Result<u64, Error> {
1093        let _pass = self.pass()?;
1094        self.store.grow_region2(&self.state, new_size)
1095    }
1096
1097    /// Whether the data area is empty (never for a provisioned region).
1098    #[must_use]
1099    pub fn is_empty(&self) -> bool {
1100        self.state.map.len() == 0
1101    }
1102}
1103
1104impl core::fmt::Debug for LiveRegion {
1105    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
1106        f.debug_struct("Region")
1107            .field("name", &self.name.as_str())
1108            .field("id", &self.id)
1109            .field("kind", &self.kind)
1110            .field("generation", &self.generation)
1111            .field("size", &self.map.len())
1112            .finish()
1113    }
1114}
1115
1116impl<P: Platform> core::fmt::Debug for AppendRegion<P> {
1117    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
1118        f.debug_tuple("AppendRegion").field(&*self.state).finish()
1119    }
1120}
1121
1122impl<P: Platform> core::fmt::Debug for PageRegion<P> {
1123    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
1124        f.debug_tuple("PageRegion").field(&*self.state).finish()
1125    }
1126}