store_io_engine/region.rs
1//! Region handles: [`AppendRegion`], [`PageRegion`] and [`Slot`].
2//!
3//! Every handle is cheap to clone and safe to share between threads. Calls
4//! block until their I/O is done.
5
6use std::ops::ControlFlow;
7use std::sync::Arc;
8use std::sync::atomic::Ordering;
9
10use store_io_core::error::{Error, FirstCause, Named, NotWrittenCause};
11use store_io_core::id::Generation;
12use store_io_format::meta::RegionKind;
13use store_io_platform::Platform;
14
15use crate::batch::{AppendBatch, PageBatch};
16use crate::gate::Pass;
17use crate::io::{ctx, ctx_range, not_written};
18use crate::lifecycle::retired_error;
19use crate::mapped::Mapped;
20use crate::options::RegionOptions;
21use crate::receipt::{DurableReceipt, RegionPos, WriteTicket};
22use crate::scan::{ScanItem, ScanSummary};
23use crate::settle::{Wrote, settle_pumped};
24use crate::store::{LiveRegion, Store};
25
26pub use crate::slot_handle::Slot;
27
28/// Region lookups and provisioning.
29impl<P: Platform> Store<P>
30where
31 P::Queue: Send,
32{
33 /// Looks up an existing append region.
34 ///
35 /// # Errors
36 ///
37 /// [`Error::NotFound`] if no ready append region has that name.
38 pub fn append_region(&self, name: &str) -> Result<AppendRegion<P>, Error> {
39 let state = self.find(name, RegionKind::Append, Named::Region)?;
40 Ok(AppendRegion {
41 store: self.clone(),
42 state,
43 })
44 }
45
46 /// Provisions a new append region of at least `size` bytes.
47 ///
48 /// # Errors
49 ///
50 /// [`Error::AlreadyExists`], [`Error::NoSpace`], or a platform error.
51 pub fn provision_append_region(&self, name: &str, size: u64) -> Result<AppendRegion<P>, Error> {
52 self.provision_append_region_with(name, RegionOptions::default().size(size))
53 }
54
55 /// Provisions a new append region as `opts` say (its size, and in
56 /// container format 2 its cap).
57 ///
58 /// # Errors
59 ///
60 /// As [`Self::provision_append_region`]; `NotWritten(InvalidOptions)`
61 /// for options [`RegionOptions::validate`] refuses; `Unsupported` for a
62 /// cap in container format 1.
63 pub fn provision_append_region_with(
64 &self,
65 name: &str,
66 opts: RegionOptions,
67 ) -> Result<AppendRegion<P>, Error> {
68 let state = self.provision(name, RegionKind::Append, opts, None)?;
69 Ok(AppendRegion {
70 store: self.clone(),
71 state,
72 })
73 }
74
75 /// Provisions a new append region with the store's
76 /// [`StoreOptions::region_defaults`](crate::StoreOptions::region_defaults).
77 ///
78 /// ```
79 /// # use std::path::Path;
80 /// # use store_io_engine::{RegionOptions, Store, StoreOptions};
81 /// # use store_io_sim::{SimConfig, SimPlatform};
82 /// let opts = StoreOptions {
83 /// region_defaults: Some(RegionOptions::default().size(1 << 20)),
84 /// ..StoreOptions::default()
85 /// };
86 /// let store = Store::create(SimPlatform::new(SimConfig::volatile(1)), Path::new("/db"), opts)?;
87 /// let log = store.provision_append_region_default("log")?;
88 /// assert_eq!(log.len(), 1 << 20);
89 /// # Ok::<(), store_io_core::error::Error>(())
90 /// ```
91 ///
92 /// # Errors
93 ///
94 /// As [`Self::provision_append_region_with`];
95 /// `NotWritten(InvalidOptions)` when the store has no defaults (before
96 /// any other refusal).
97 pub fn provision_append_region_default(&self, name: &str) -> Result<AppendRegion<P>, Error> {
98 self.provision_append_region_with(name, self.region_defaults()?)
99 }
100
101 /// Provisions a new page region with the store's
102 /// [`StoreOptions::region_defaults`](crate::StoreOptions::region_defaults).
103 ///
104 /// # Errors
105 ///
106 /// As [`Self::provision_append_region_default`].
107 pub fn provision_page_region_default(&self, name: &str) -> Result<PageRegion<P>, Error> {
108 self.provision_page_region_with(name, self.region_defaults()?)
109 }
110
111 /// The store's region defaults; refused when it has none (before any
112 /// I/O, and before the store's own refusals: read-only, poisoned).
113 pub(crate) fn region_defaults(&self) -> Result<RegionOptions, Error> {
114 self.inner.opts.region_defaults.ok_or(Error::NotWritten {
115 cause: NotWrittenCause::InvalidOptions,
116 ctx: store_io_core::error::ErrorContext::default(),
117 })
118 }
119
120 /// Looks up an existing page region.
121 ///
122 /// # Errors
123 ///
124 /// [`Error::NotFound`] if no ready page region has that name.
125 pub fn page_region(&self, name: &str) -> Result<PageRegion<P>, Error> {
126 let state = self.find(name, RegionKind::Page, Named::Region)?;
127 Ok(PageRegion {
128 store: self.clone(),
129 state,
130 })
131 }
132
133 /// Provisions a new page region of at least `size` bytes.
134 ///
135 /// # Errors
136 ///
137 /// [`Error::AlreadyExists`], [`Error::NoSpace`], or a platform error.
138 pub fn provision_page_region(&self, name: &str, size: u64) -> Result<PageRegion<P>, Error> {
139 self.provision_page_region_with(name, RegionOptions::default().size(size))
140 }
141
142 /// Provisions a new page region as `opts` say (its size, and in
143 /// container format 2 its cap).
144 ///
145 /// # Errors
146 ///
147 /// As [`Self::provision_append_region_with`].
148 pub fn provision_page_region_with(
149 &self,
150 name: &str,
151 opts: RegionOptions,
152 ) -> Result<PageRegion<P>, Error> {
153 let state = self.provision(name, RegionKind::Page, opts, None)?;
154 Ok(PageRegion {
155 store: self.clone(),
156 state,
157 })
158 }
159
160 /// Looks up an existing small-object slot.
161 ///
162 /// # Errors
163 ///
164 /// [`Error::NotFound`] if no slot has that name.
165 pub fn slot(&self, name: &str) -> Result<Slot<P>, Error> {
166 let state = self.find(name, RegionKind::Slot, Named::Slot)?;
167 Ok(Slot::from_parts(self.clone(), state))
168 }
169
170 /// Provisions a new small-object slot. Its capacity is the block size
171 /// minus 176 bytes of headers ([`Slot::capacity`]).
172 ///
173 /// # Errors
174 ///
175 /// [`Error::AlreadyExists`], [`Error::NoSpace`], or a platform error.
176 pub fn provision_slot(&self, name: &str) -> Result<Slot<P>, Error> {
177 let state = self.provision(name, RegionKind::Slot, RegionOptions::default(), None)?;
178 Ok(Slot::from_parts(self.clone(), state))
179 }
180}
181
182// ---------------------------------------------------------------------------
183// Append regions
184// ---------------------------------------------------------------------------
185
186/// A region store-io appends to. Positions are chosen by store-io from an
187/// atomic tail, so appends from many threads never overlap and a written
188/// block is never rewritten.
189///
190/// Each append occupies whole blocks: the bytes from the end of the data to
191/// the next block boundary are zero and are not caller data. Pack small
192/// records into one append (or use a batch) to avoid the padding.
193pub struct AppendRegion<P: Platform> {
194 store: Store<P>,
195 state: Arc<LiveRegion>,
196}
197
198impl<P: Platform> AppendRegion<P> {
199 pub(crate) fn from_parts(store: Store<P>, state: Arc<LiveRegion>) -> Self {
200 Self { store, state }
201 }
202}
203
204impl<P: Platform> Clone for AppendRegion<P> {
205 fn clone(&self) -> Self {
206 Self {
207 store: self.store.clone(),
208 state: Arc::clone(&self.state),
209 }
210 }
211}
212
213impl<P: Platform> AppendRegion<P>
214where
215 P::Queue: Send,
216{
217 /// Threads waiting on the region's completed prefix (an append waiting
218 /// for room in the window, a sync waiting for its appends). For tests.
219 #[doc(hidden)]
220 #[must_use]
221 pub fn waiting_on_appends(&self) -> u64 {
222 self.state.frontier.as_ref().map_or(0, |f| f.waiting())
223 }
224
225 fn frontier(&self) -> Result<&crate::frontier::AppendFrontier, Error> {
226 self.state
227 .frontier
228 .as_ref()
229 .ok_or_else(|| not_written(NotWrittenCause::NotReady, ctx(&self.state)))
230 }
231
232 /// The checks every append makes before touching the device, and the
233 /// pass that keeps a recycle or release out until the append is done.
234 pub(crate) fn writable(&self) -> Result<(&crate::frontier::AppendFrontier, Pass<'_>), Error> {
235 let pass = self.pass()?;
236 self.store.ready_for(&self.state)?;
237 let f = self.frontier()?;
238 if !self.state.positioned.load(Ordering::Acquire) {
239 return Err(not_written(
240 NotWrittenCause::NotPositioned,
241 ctx(&self.state),
242 ));
243 }
244 Ok((f, pass))
245 }
246
247 fn pass(&self) -> Result<Pass<'_>, Error> {
248 self.state
249 .gate
250 .enter()
251 .map_err(|why| retired_error(why, &self.state))
252 }
253
254 /// The generation this handle writes and reads. Recycling starts the
255 /// next one; embed it in your records to tell generations apart after a
256 /// crash.
257 #[must_use]
258 pub fn generation(&self) -> Generation {
259 self.state.generation
260 }
261
262 /// Starts a new generation: waits for every operation in flight on this
263 /// region, rewrites the region header with the next generation, makes it
264 /// durable, and returns a handle to the new generation with its append
265 /// position at 0. No other I/O: the old bytes stay on disk, so records
266 /// must carry their generation. Every handle, ticket and position of the
267 /// old generation is refused from then on (`StaleGeneration`).
268 ///
269 /// Must not be called from inside this region's own scan visitor, nor by
270 /// a thread whose own [`crate::Submitter`] has operations on this region
271 /// it has not reaped: recycle waits for them, and only that thread's
272 /// `reap` or `wait` finishes them (the same holds for `release`).
273 ///
274 /// # Errors
275 ///
276 /// `NotWritten(StaleGeneration | NotReady)` if the region was already
277 /// recycled or released; `NotWritten(ReadOnly)` if the store or the
278 /// region is read-only; `Io` if the store's queue could not be prepared
279 /// (nothing was written; the region stays usable); `Poisoned`;
280 /// `DurabilityUnknown`.
281 pub fn recycle(&self) -> Result<Self, Error> {
282 let state = self.store.recycle_region(&self.state)?;
283 Ok(Self {
284 store: self.store.clone(),
285 state,
286 })
287 }
288
289 /// Releases the region: waits for every operation in flight, records
290 /// the release in the region table (durable: from then on the release
291 /// stands), marks its header released (durable), and gives its space
292 /// back to the file system. Its contents are undefined from then on;
293 /// every handle to it is refused (`NotReady`), and its space is reused
294 /// by later provisioning. A header write that fails before it reaches
295 /// the device (refused, a queue that could not be prepared, a header
296 /// pair that would not encode or plan) still returns success: the space
297 /// is free for provisioning, but not given back to the file system. Not
298 /// from a thread with unreaped [`crate::Submitter`] operations on it
299 /// (see [`Self::recycle`]).
300 ///
301 /// # Errors
302 ///
303 /// `NotWritten(StaleGeneration | NotReady)` if the region was already
304 /// recycled or released; `NotWritten(ReadOnly)` if the store or the
305 /// region is read-only; on format 2, `NotWritten(Frozen)` while a table
306 /// page is stale or lost, and `NotWritten(Overlap)` when removing the
307 /// entry would change another region's verdict at the next open;
308 /// `Poisoned`; `DurabilityUnknown`; or `Io` if the space could not be
309 /// deallocated (the region is released either way). Once the table
310 /// records the release, a release fails only on a poisoned store or
311 /// with that `Io`.
312 pub fn release(self) -> Result<(), Error> {
313 self.store.release_region(&self.state)
314 }
315
316 pub(crate) fn store(&self) -> &Store<P> {
317 &self.store
318 }
319
320 pub(crate) fn state(&self) -> &Arc<LiveRegion> {
321 &self.state
322 }
323
324 /// Appends `data` (any length of at least one byte) and returns a ticket
325 /// once the write has completed. The data is not yet durable: pass the
326 /// ticket to [`Self::sync_through`], or use [`Self::append_durable`].
327 ///
328 /// The append starts at a block boundary chosen by store-io and is
329 /// zero-padded to the next one. Data larger than the largest pooled
330 /// buffer is written in several pieces, kept in flight together.
331 ///
332 /// # Errors
333 ///
334 /// `NotWritten` (empty data, region full, pool exhausted, read-only, not
335 /// positioned), `Io` if the store's queue could not be prepared (nothing
336 /// was reserved or written), `Poisoned`, or `DurabilityUnknown` (the
337 /// domain is now poisoned).
338 pub fn append(&self, data: &[u8]) -> Result<WriteTicket, Error> {
339 let s = &self.store;
340 let (f, _pass) = self.writable()?;
341 let c = ctx(&self.state);
342 if data.is_empty() {
343 return Err(not_written(NotWrittenCause::Empty, c));
344 }
345 // Everything that can fail without touching the device happens first:
346 // the first piece is staged before any space is reserved.
347 let mut stream = s.stage(data, c)?;
348 let (mut lane, reserved) = s.lane_and_reservation(&self.state, f, data.len() as u64, c)?;
349 let r = reserved.range();
350 stream.at(r.start);
351 // Until the pump returns the guard poisons on any exit (a panic
352 // included); then the settle step completes the reservation, or
353 // poisons: past it every failure leaves a hole.
354 let outcome = s.pump(&mut lane, &mut Mapped::new(&self.state.map, &mut stream));
355 let r = reserved.disarm();
356 settle_pumped(s, &self.state, Wrote::Append(r), outcome)
357 }
358
359 /// Makes everything appended up to and including `ticket` durable.
360 ///
361 /// # Errors
362 ///
363 /// `NotWritten(ForeignPosition | StaleGeneration)` for a ticket of another
364 /// region or generation; `Poisoned`; `DurabilityUnknown` if the flush
365 /// failed.
366 pub fn sync_through(&self, ticket: &WriteTicket) -> Result<DurableReceipt, Error> {
367 let s = &self.store;
368 let f = self.frontier()?;
369 let c = ctx(&self.state);
370 self.state
371 .gate
372 .check()
373 .map_err(|why| retired_error(why, &self.state))?;
374 if ticket.epoch != s.inner.epoch
375 || ticket.volume != s.inner.volume
376 || ticket.region != self.state.id
377 {
378 return Err(not_written(NotWrittenCause::ForeignPosition, c));
379 }
380 if ticket.generation != self.state.generation {
381 return Err(not_written(NotWrittenCause::StaleGeneration, c));
382 }
383 // Already durable: no barrier. A forced region on a power-safe store
384 // counts only what a data flush made durable (D-102).
385 let flushes = s.inner.policy.flush;
386 let synced = flushes || self.state.forced();
387 let durable = if synced && !flushes {
388 f.synced()
389 } else {
390 f.durable()
391 };
392 if durable >= ticket.end {
393 s.inner.domain.count_already_durable();
394 return Ok(s.receipt(&self.state, 0, durable, 0, true, synced));
395 }
396 let domain = &s.inner.domain;
397 if !f.wait_completed(ticket.end, || domain.poisoned().is_some()) {
398 return Err(Error::Poisoned {
399 first: domain.poisoned().unwrap_or(FirstCause {
400 op: None,
401 raw: None,
402 }),
403 });
404 }
405 // Snapshot the completed prefix BEFORE the barrier takes its ticket.
406 let through = f.completed();
407 let (need, synced) = s.barrier_for(&self.state)?;
408 if synced {
409 f.publish_synced(through);
410 } else {
411 f.publish_durable(through);
412 }
413 Ok(s.receipt(&self.state, 0, through, need, true, synced))
414 }
415
416 /// Appends `data` and makes it durable before returning.
417 ///
418 /// # Errors
419 ///
420 /// As [`Self::append`] and [`Self::sync_through`].
421 pub fn append_durable(&self, data: &[u8]) -> Result<(RegionPos, DurableReceipt), Error> {
422 let t = self.append(data)?;
423 let pos = t.pos();
424 let r = self.sync_through(&t)?;
425 Ok((pos, r))
426 }
427
428 /// A batch: many records packed back to back, written together and made
429 /// durable with one barrier. See [`AppendBatch`].
430 #[must_use]
431 pub fn batch(&self) -> AppendBatch<P> {
432 AppendBatch::new(self.clone())
433 }
434
435 /// Reads raw bytes at a block-aligned offset into `out` (any length).
436 /// Returns the bytes read, fewer than `out.len()` only at the end of the
437 /// region's data area.
438 ///
439 /// The bytes are read into store-io's own aligned buffers and copied
440 /// into `out`: direct I/O needs aligned memory, and a read whose wait
441 /// fails stays in flight, so it must never target the caller's memory.
442 /// To read with no copy, use [`Self::scan`] (it lends store-io's
443 /// buffers) or a [`crate::Submitter`]'s `read_append` (it reads into a
444 /// buffer taken from the store).
445 ///
446 /// # Errors
447 ///
448 /// `NotWritten` for a misaligned or out-of-range read;
449 /// `Corruption(MediaError)` if the device reports an unreadable range,
450 /// with the exact range in the error's context; `Io` with the raw code
451 /// for any other failure.
452 pub fn read(&self, offset: u64, out: &mut [u8]) -> Result<usize, Error> {
453 let _pass = self.pass()?;
454 self.store.read_at(&self.state, offset, out)
455 }
456
457 /// Reads the region in order from the block-aligned offset `from` to the
458 /// end of its data area, handing every piece to `visit`; return
459 /// [`ControlFlow::Break`] to stop (typically where your valid data ends).
460 ///
461 /// The scan reads with direct I/O and several large reads in flight. It
462 /// never stops at bad data: a range the device cannot read is narrowed
463 /// to whole sectors (the device's logical blocks where the probe
464 /// reported the direct-I/O alignment, else store blocks), reported as
465 /// [`ScanItem::Unreadable`], and skipped.
466 /// Scans work on read-only and poisoned stores.
467 ///
468 /// `visit` may call the store (read, write, provision): it never runs
469 /// while the scan holds a queue another call waits for (on a shared
470 /// queue the scan puts it back before delivering). This region's
471 /// `recycle` and `release` wait for the scan itself to end, so they
472 /// must not be called from `visit`; nor may an append wait there for
473 /// room that only the same thread's [`crate::Submitter`] can free.
474 ///
475 /// # Errors
476 ///
477 /// `NotWritten(Misaligned | OutOfBounds | PoolExhausted)` before any I/O;
478 /// `Io` if a read fails for a reason other than the media.
479 pub fn scan(
480 &self,
481 from: u64,
482 visit: impl FnMut(ScanItem<'_>) -> ControlFlow<()>,
483 ) -> Result<ScanSummary, Error> {
484 let _pass = self.pass()?;
485 self.store
486 .scan_reported(&self.state, from, self.state.map.len(), visit)
487 }
488
489 /// Moves the append position forward to `offset` (rounded up to a block),
490 /// typically after recovery found where valid data ends. Never moves
491 /// backwards, and only while no append is in flight.
492 ///
493 /// The skipped range counts as durable: a writable open flushed the
494 /// device before reading anything, so whatever recovery found is on
495 /// media. Bytes appended in this open and not yet made durable never do:
496 /// then the durable point stays where it is, and the next sync that
497 /// flushes covers both (`durable_through` reads lower until then).
498 /// A read-only open flushes nothing, so there `resume_at` is refused
499 /// (D-072); appends are refused there anyway, and so is a poisoned
500 /// store.
501 ///
502 /// # Errors
503 ///
504 /// `NotWritten(ReadOnly | TooManyInFlight | OutOfBounds)`, `Poisoned`.
505 pub fn resume_at(&self, offset: u64) -> Result<u64, Error> {
506 if self.store.inner.read_only || self.state.is_read_only() {
507 return Err(Error::NotWritten {
508 cause: NotWrittenCause::ReadOnly,
509 ctx: ctx(&self.state),
510 });
511 }
512 // A poison landing after this check is harmless: a failed append
513 // leaves the prefix below the tail (refused below), a failed flush
514 // leaves the durable point below it (not raised).
515 if let Some(first) = self.store.inner.domain.poisoned() {
516 return Err(Error::Poisoned { first });
517 }
518 let _pass = self.pass()?;
519 let at = self
520 .frontier()?
521 .resume_at(offset)
522 .map_err(|cause| Error::NotWritten {
523 cause,
524 ctx: ctx(&self.state),
525 })?;
526 self.state
527 .positioned
528 .store(true, std::sync::atomic::Ordering::Release);
529 Ok(at)
530 }
531
532 /// Where the next append will start.
533 #[must_use]
534 pub fn tail(&self) -> u64 {
535 self.state
536 .frontier
537 .as_ref()
538 .map_or(0, crate::frontier::AppendFrontier::tail)
539 }
540
541 /// Every byte before this offset is durable.
542 #[must_use]
543 pub fn durable_through(&self) -> u64 {
544 self.state
545 .frontier
546 .as_ref()
547 .map_or(0, crate::frontier::AppendFrontier::durable)
548 }
549
550 /// Data-area size in bytes.
551 #[must_use]
552 pub fn len(&self) -> u64 {
553 self.state.map.len()
554 }
555
556 /// How far the data area can grow without new space: the region's
557 /// capacity in bytes (the sum of its extents).
558 #[must_use]
559 pub fn capacity(&self) -> u64 {
560 self.state.map.capacity()
561 }
562
563 /// Grows the data area to at least `new_size` bytes (rounded up to the
564 /// block size): the new range is filled, flushed and verified before
565 /// one table page write commits the new size (spec §12). Within the
566 /// capacity nothing else moves; past it the region gains one extent of
567 /// at least 1 MiB and an eighth of its capacity (after its last one
568 /// when that space is free), listed in its header before the commit.
569 /// Returns the new size; on a writable store a size at most the current
570 /// one writes nothing and returns it. Positions and tickets taken
571 /// before stay valid.
572 ///
573 /// # Errors
574 ///
575 /// `Unsupported` in container format 1; `NotWritten` (`TooLarge` past
576 /// 2^32 blocks; `ReadOnly`; `Frozen` while a table page is stale or
577 /// lost; `NotReady` if the filled range does not verify;
578 /// `StaleGeneration` for a recycled region; `QueueFull`); `NoSpace`
579 /// (`RegionLimit` for a capped region, `Extents` when its header lists
580 /// as many extents as it can hold, `Caller` past the tag's cap,
581 /// `Growth` when the device runs out); `Io` if the range could not be
582 /// probed or allocated; `Corruption` for metadata that does not decode;
583 /// `Poisoned`; `DurabilityUnknown` (a fill, a flush, the header or the
584 /// page write failed after submission: poisoned).
585 pub fn grow(&self, new_size: u64) -> Result<u64, Error> {
586 let _pass = self.pass()?;
587 self.store.grow_region2(&self.state, new_size)
588 }
589
590 /// Whether the data area is empty (never for a provisioned region).
591 #[must_use]
592 pub fn is_empty(&self) -> bool {
593 self.state.map.len() == 0
594 }
595
596 /// Block size: appends are padded to it and reads must be aligned to it.
597 #[must_use]
598 pub fn block_size(&self) -> u64 {
599 self.store.inner.layout.block()
600 }
601}
602
603// ---------------------------------------------------------------------------
604// Page regions
605// ---------------------------------------------------------------------------
606
607/// A region of caller-addressed blocks, overwritten in place.
608pub struct PageRegion<P: Platform> {
609 store: Store<P>,
610 state: Arc<LiveRegion>,
611}
612
613impl<P: Platform> PageRegion<P> {
614 pub(crate) fn from_parts(store: Store<P>, state: Arc<LiveRegion>) -> Self {
615 Self { store, state }
616 }
617}
618
619impl<P: Platform> Clone for PageRegion<P> {
620 fn clone(&self) -> Self {
621 Self {
622 store: self.store.clone(),
623 state: Arc::clone(&self.state),
624 }
625 }
626}
627
628impl<P: Platform> PageRegion<P>
629where
630 P::Queue: Send,
631{
632 /// A checked position: `offset` must be block-aligned and inside the
633 /// region.
634 ///
635 /// # Errors
636 ///
637 /// `NotWritten(Misaligned | OutOfBounds)`.
638 pub fn pos(&self, offset: u64) -> Result<RegionPos, Error> {
639 let c = ctx(&self.state);
640 if offset % self.store.block() != 0 {
641 return Err(not_written(NotWrittenCause::Misaligned, c));
642 }
643 if offset >= self.state.map.len() {
644 return Err(not_written(NotWrittenCause::OutOfBounds, c));
645 }
646 Ok(RegionPos {
647 volume: self.store.inner.volume,
648 region: self.state.id,
649 generation: self.state.generation,
650 offset,
651 })
652 }
653
654 /// Checks a write of `len` bytes at `pos`; returns its end.
655 pub(crate) fn check(&self, pos: &RegionPos, len: usize) -> Result<u64, Error> {
656 let c = ctx(&self.state);
657 if pos.volume != self.store.inner.volume || pos.region != self.state.id {
658 return Err(not_written(NotWrittenCause::ForeignPosition, c));
659 }
660 if pos.generation != self.state.generation {
661 return Err(not_written(NotWrittenCause::StaleGeneration, c));
662 }
663 if len == 0 {
664 return Err(not_written(NotWrittenCause::Empty, c));
665 }
666 if (len as u64) % self.store.block() != 0 {
667 return Err(not_written(NotWrittenCause::Misaligned, c));
668 }
669 match pos.offset.checked_add(len as u64) {
670 Some(end) if end <= self.state.map.len() => Ok(end),
671 _ => Err(not_written(NotWrittenCause::OutOfBounds, c)),
672 }
673 }
674
675 pub(crate) fn store(&self) -> &Store<P> {
676 &self.store
677 }
678
679 pub(crate) fn state(&self) -> &Arc<LiveRegion> {
680 &self.state
681 }
682
683 pub(crate) fn pass(&self) -> Result<Pass<'_>, Error> {
684 self.state
685 .gate
686 .enter()
687 .map_err(|why| retired_error(why, &self.state))
688 }
689
690 /// The generation this handle writes and reads.
691 #[must_use]
692 pub fn generation(&self) -> Generation {
693 self.state.generation
694 }
695
696 /// Starts a new generation (see [`AppendRegion::recycle`]); positions and
697 /// tickets of the old generation are refused from then on.
698 ///
699 /// # Errors
700 ///
701 /// As [`AppendRegion::recycle`].
702 pub fn recycle(&self) -> Result<Self, Error> {
703 let state = self.store.recycle_region(&self.state)?;
704 Ok(Self {
705 store: self.store.clone(),
706 state,
707 })
708 }
709
710 /// Releases the region (see [`AppendRegion::release`]).
711 ///
712 /// # Errors
713 ///
714 /// As [`AppendRegion::release`].
715 pub fn release(self) -> Result<(), Error> {
716 self.store.release_region(&self.state)
717 }
718
719 /// Writes whole blocks at `pos` and returns a ticket once the write has
720 /// completed. `data.len()` must be a multiple of [`Self::block_size`];
721 /// any size is accepted (large writes go out in several pieces).
722 ///
723 /// # Errors
724 ///
725 /// `NotWritten` (foreign, stale, misaligned, out of range, pool
726 /// exhausted), `Io` (refused by the platform before reaching the device;
727 /// nothing is poisoned), `Poisoned`, or `DurabilityUnknown`.
728 pub fn write(&self, pos: RegionPos, data: &[u8]) -> Result<WriteTicket, Error> {
729 let s = &self.store;
730 let _pass = self.pass()?;
731 s.ready_for(&self.state)?;
732 let end = self.check(&pos, data.len())?;
733 let c = ctx_range(&self.state, pos.offset, end);
734 let outcome = s.write_staged(&self.state, pos.offset, data, c)?;
735 let wrote = Wrote::Page {
736 start: pos.offset,
737 end,
738 };
739 settle_pumped(s, &self.state, wrote, outcome)
740 }
741
742 /// Makes the ticket's write durable, together with every other write to
743 /// this store that completed before the call.
744 ///
745 /// # Errors
746 ///
747 /// `NotWritten(ForeignPosition | StaleGeneration)`, `Poisoned`, or
748 /// `DurabilityUnknown`.
749 pub fn sync_through(&self, ticket: &WriteTicket) -> Result<DurableReceipt, Error> {
750 let s = &self.store;
751 let c = ctx(&self.state);
752 if ticket.epoch != s.inner.epoch
753 || ticket.volume != s.inner.volume
754 || ticket.region != self.state.id
755 {
756 return Err(not_written(NotWrittenCause::ForeignPosition, c));
757 }
758 if ticket.generation != self.state.generation {
759 return Err(not_written(NotWrittenCause::StaleGeneration, c));
760 }
761 self.state
762 .gate
763 .check()
764 .map_err(|why| retired_error(why, &self.state))?;
765 let (need, synced) = s.barrier_for(&self.state)?;
766 Ok(s.receipt(&self.state, ticket.start, ticket.end, need, false, synced))
767 }
768
769 /// Writes whole blocks at `pos` and makes them durable before returning.
770 ///
771 /// # Errors
772 ///
773 /// As [`Self::write`] and [`Self::sync_through`].
774 pub fn write_durable(&self, pos: RegionPos, data: &[u8]) -> Result<DurableReceipt, Error> {
775 let t = self.write(pos, data)?;
776 self.sync_through(&t)
777 }
778
779 /// A batch: many page writes submitted together and made durable with
780 /// one barrier. See [`PageBatch`].
781 #[must_use]
782 pub fn batch(&self) -> PageBatch<P> {
783 PageBatch::new(self.clone())
784 }
785
786 /// Reads at `pos` into `out` (any length; reads whole blocks internally).
787 /// Returns the bytes read, fewer than `out.len()` only at the end of the
788 /// region.
789 ///
790 /// The bytes are read into store-io's own aligned buffers and copied
791 /// into `out`: direct I/O needs aligned memory, and a read whose wait
792 /// fails stays in flight, so it must never target the caller's memory.
793 /// To read with no copy, use [`Self::scan`] (it lends store-io's
794 /// buffers) or a [`crate::Submitter`]'s `read_page` (it reads into a
795 /// buffer taken from the store).
796 ///
797 /// # Errors
798 ///
799 /// `NotWritten` for a foreign, stale or out-of-range position;
800 /// `Corruption(MediaError)` for an unreadable range, with the exact range
801 /// in the error's context; `Io` with the raw code for any other failure.
802 pub fn read(&self, pos: RegionPos, out: &mut [u8]) -> Result<usize, Error> {
803 let c = ctx(&self.state);
804 if pos.volume != self.store.inner.volume || pos.region != self.state.id {
805 return Err(not_written(NotWrittenCause::ForeignPosition, c));
806 }
807 if pos.generation != self.state.generation {
808 return Err(not_written(NotWrittenCause::StaleGeneration, c));
809 }
810 let _pass = self.pass()?;
811 self.store.read_at(&self.state, pos.offset, out)
812 }
813
814 /// Reads the region in order from the block-aligned offset `from` to the
815 /// end of its data area, handing every piece to `visit`; return
816 /// [`ControlFlow::Break`] to stop (typically where your valid data ends).
817 ///
818 /// The scan reads with direct I/O and several large reads in flight. It
819 /// never stops at bad data: a range the device cannot read is narrowed
820 /// to whole sectors (the device's logical blocks where the probe
821 /// reported the direct-I/O alignment, else store blocks), reported as
822 /// [`ScanItem::Unreadable`], and skipped.
823 /// Scans work on read-only and poisoned stores.
824 ///
825 /// `visit` may call the store (read, write, provision): it never runs
826 /// while the scan holds a queue another call waits for (on a shared
827 /// queue the scan puts it back before delivering). This region's
828 /// `recycle` and `release` wait for the scan itself to end, so they
829 /// must not be called from `visit`; nor may an append wait there for
830 /// room that only the same thread's [`crate::Submitter`] can free.
831 ///
832 /// # Errors
833 ///
834 /// `NotWritten(Misaligned | OutOfBounds | PoolExhausted)` before any I/O;
835 /// `Io` if a read fails for a reason other than the media.
836 pub fn scan(
837 &self,
838 from: u64,
839 visit: impl FnMut(ScanItem<'_>) -> ControlFlow<()>,
840 ) -> Result<ScanSummary, Error> {
841 let _pass = self.pass()?;
842 self.store
843 .scan_reported(&self.state, from, self.state.map.len(), visit)
844 }
845
846 /// Block size: writes must be whole blocks.
847 #[must_use]
848 pub fn block_size(&self) -> u64 {
849 self.store.inner.layout.block()
850 }
851
852 /// Data-area size in bytes.
853 #[must_use]
854 pub fn len(&self) -> u64 {
855 self.state.map.len()
856 }
857
858 /// How far the data area can grow without new space: the region's
859 /// capacity in bytes (the sum of its extents).
860 #[must_use]
861 pub fn capacity(&self) -> u64 {
862 self.state.map.capacity()
863 }
864
865 /// Grows the data area to at least `new_size` bytes (rounded up to the
866 /// block size): the new range is filled, flushed and verified before
867 /// one table page write commits the new size (spec §12). Within the
868 /// capacity nothing else moves; past it the region gains one extent of
869 /// at least 1 MiB and an eighth of its capacity (after its last one
870 /// when that space is free), listed in its header before the commit.
871 /// Returns the new size; on a writable store a size at most the current
872 /// one writes nothing and returns it. Positions and tickets taken
873 /// before stay valid.
874 ///
875 /// # Errors
876 ///
877 /// `Unsupported` in container format 1; `NotWritten` (`TooLarge` past
878 /// 2^32 blocks; `ReadOnly`; `Frozen` while a table page is stale or
879 /// lost; `NotReady` if the filled range does not verify;
880 /// `StaleGeneration` for a recycled region; `QueueFull`); `NoSpace`
881 /// (`RegionLimit` for a capped region, `Extents` when its header lists
882 /// as many extents as it can hold, `Caller` past the tag's cap,
883 /// `Growth` when the device runs out); `Io` if the range could not be
884 /// probed or allocated; `Corruption` for metadata that does not decode;
885 /// `Poisoned`; `DurabilityUnknown` (a fill, a flush, the header or the
886 /// page write failed after submission: poisoned).
887 pub fn grow(&self, new_size: u64) -> Result<u64, Error> {
888 let _pass = self.pass()?;
889 self.store.grow_region2(&self.state, new_size)
890 }
891
892 /// Whether the data area is empty (never for a provisioned region).
893 #[must_use]
894 pub fn is_empty(&self) -> bool {
895 self.state.map.len() == 0
896 }
897}
898
899impl core::fmt::Debug for LiveRegion {
900 fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
901 f.debug_struct("Region")
902 .field("name", &self.name.as_str())
903 .field("id", &self.id)
904 .field("kind", &self.kind)
905 .field("generation", &self.generation)
906 .field("size", &self.map.len())
907 .finish()
908 }
909}
910
911impl<P: Platform> core::fmt::Debug for AppendRegion<P> {
912 fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
913 f.debug_tuple("AppendRegion").field(&*self.state).finish()
914 }
915}
916
917impl<P: Platform> core::fmt::Debug for PageRegion<P> {
918 fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
919 f.debug_tuple("PageRegion").field(&*self.state).finish()
920 }
921}