store_io_engine/region.rs
1//! Region handles: [`AppendRegion`], [`PageRegion`] and [`Slot`].
2//!
3//! Every handle is cheap to clone and safe to share between threads. Calls
4//! block until their I/O is done.
5
6use std::ops::ControlFlow;
7use std::sync::Arc;
8use std::sync::atomic::Ordering;
9
10use store_io_core::error::{Error, FirstCause, Named, NotWrittenCause};
11use store_io_core::id::Generation;
12use store_io_format::meta::RegionKind;
13use store_io_platform::Platform;
14
15use crate::batch::{AppendBatch, PageBatch};
16use crate::gate::Pass;
17use crate::io::{ctx, ctx_range, not_written};
18use crate::lifecycle::retired_error;
19use crate::mapped::Mapped;
20use crate::options::RegionOptions;
21use crate::receipt::{DurableReceipt, RegionPos, WriteTicket};
22use crate::scan::{ScanItem, ScanSummary};
23use crate::settle::{Wrote, settle_pumped};
24use crate::store::{LiveRegion, Store};
25
26pub use crate::slot_handle::Slot;
27
28/// Region lookups and provisioning.
29impl<P: Platform> Store<P>
30where
31 P::Queue: Send,
32{
33 /// Looks up an existing append region.
34 ///
35 /// # Errors
36 ///
37 /// [`Error::NotFound`] if no ready append region has that name.
38 pub fn append_region(&self, name: &str) -> Result<AppendRegion<P>, Error> {
39 let state = self.find(name, RegionKind::Append, Named::Region)?;
40 Ok(AppendRegion {
41 store: self.clone(),
42 state,
43 })
44 }
45
46 /// Provisions a new append region of at least `size` bytes.
47 ///
48 /// # Errors
49 ///
50 /// [`Error::AlreadyExists`], [`Error::NoSpace`], or a platform error.
51 pub fn provision_append_region(&self, name: &str, size: u64) -> Result<AppendRegion<P>, Error> {
52 self.provision_append_region_with(name, RegionOptions::default().size(size))
53 }
54
55 /// Provisions a new append region as `opts` say (its size, and in
56 /// container format 2 its cap).
57 ///
58 /// # Errors
59 ///
60 /// As [`Self::provision_append_region`]; `NotWritten(InvalidOptions)`
61 /// for options [`RegionOptions::validate`] refuses; `Unsupported` for a
62 /// cap in container format 1.
63 pub fn provision_append_region_with(
64 &self,
65 name: &str,
66 opts: RegionOptions,
67 ) -> Result<AppendRegion<P>, Error> {
68 let state = self.provision(name, RegionKind::Append, opts, None)?;
69 Ok(AppendRegion {
70 store: self.clone(),
71 state,
72 })
73 }
74
75 /// Provisions a new append region with the store's
76 /// [`StoreOptions::region_defaults`](crate::StoreOptions::region_defaults).
77 ///
78 /// ```
79 /// # use std::path::Path;
80 /// # use store_io_engine::{RegionOptions, Store, StoreOptions};
81 /// # use store_io_sim::{SimConfig, SimPlatform};
82 /// let opts = StoreOptions {
83 /// region_defaults: Some(RegionOptions::default().size(1 << 20)),
84 /// ..StoreOptions::default()
85 /// };
86 /// let store = Store::create(SimPlatform::new(SimConfig::volatile(1)), Path::new("/db"), opts)?;
87 /// let log = store.provision_append_region_default("log")?;
88 /// assert_eq!(log.len(), 1 << 20);
89 /// # Ok::<(), store_io_core::error::Error>(())
90 /// ```
91 ///
92 /// # Errors
93 ///
94 /// As [`Self::provision_append_region_with`];
95 /// `NotWritten(InvalidOptions)` when the store has no defaults (before
96 /// any other refusal).
97 pub fn provision_append_region_default(&self, name: &str) -> Result<AppendRegion<P>, Error> {
98 self.provision_append_region_with(name, self.region_defaults()?)
99 }
100
101 /// Provisions a new page region with the store's
102 /// [`StoreOptions::region_defaults`](crate::StoreOptions::region_defaults).
103 ///
104 /// # Errors
105 ///
106 /// As [`Self::provision_append_region_default`].
107 pub fn provision_page_region_default(&self, name: &str) -> Result<PageRegion<P>, Error> {
108 self.provision_page_region_with(name, self.region_defaults()?)
109 }
110
111 /// The store's region defaults; refused when it has none (before any
112 /// I/O, and before the store's own refusals: read-only, poisoned).
113 pub(crate) fn region_defaults(&self) -> Result<RegionOptions, Error> {
114 self.inner.opts.region_defaults.ok_or(Error::NotWritten {
115 cause: NotWrittenCause::InvalidOptions,
116 ctx: store_io_core::error::ErrorContext::default(),
117 })
118 }
119
120 /// Looks up an existing page region.
121 ///
122 /// # Errors
123 ///
124 /// [`Error::NotFound`] if no ready page region has that name.
125 pub fn page_region(&self, name: &str) -> Result<PageRegion<P>, Error> {
126 let state = self.find(name, RegionKind::Page, Named::Region)?;
127 Ok(PageRegion {
128 store: self.clone(),
129 state,
130 })
131 }
132
133 /// Provisions a new page region of at least `size` bytes.
134 ///
135 /// # Errors
136 ///
137 /// [`Error::AlreadyExists`], [`Error::NoSpace`], or a platform error.
138 pub fn provision_page_region(&self, name: &str, size: u64) -> Result<PageRegion<P>, Error> {
139 self.provision_page_region_with(name, RegionOptions::default().size(size))
140 }
141
142 /// Provisions a new page region as `opts` say (its size, and in
143 /// container format 2 its cap).
144 ///
145 /// # Errors
146 ///
147 /// As [`Self::provision_append_region_with`].
148 pub fn provision_page_region_with(
149 &self,
150 name: &str,
151 opts: RegionOptions,
152 ) -> Result<PageRegion<P>, Error> {
153 let state = self.provision(name, RegionKind::Page, opts, None)?;
154 Ok(PageRegion {
155 store: self.clone(),
156 state,
157 })
158 }
159
160 /// Looks up an existing small-object slot.
161 ///
162 /// # Errors
163 ///
164 /// [`Error::NotFound`] if no slot has that name.
165 pub fn slot(&self, name: &str) -> Result<Slot<P>, Error> {
166 let state = self.find(name, RegionKind::Slot, Named::Slot)?;
167 Ok(Slot::from_parts(self.clone(), state))
168 }
169
170 /// Provisions a new small-object slot. Its capacity is the block size
171 /// minus 176 bytes of headers ([`Slot::capacity`]).
172 ///
173 /// # Errors
174 ///
175 /// [`Error::AlreadyExists`], [`Error::NoSpace`], or a platform error.
176 pub fn provision_slot(&self, name: &str) -> Result<Slot<P>, Error> {
177 let state = self.provision(name, RegionKind::Slot, RegionOptions::default(), None)?;
178 Ok(Slot::from_parts(self.clone(), state))
179 }
180}
181
182// ---------------------------------------------------------------------------
183// Append regions
184// ---------------------------------------------------------------------------
185
186/// A region store-io appends to. Positions are chosen by store-io from an
187/// atomic tail, so appends from many threads never overlap and a written
188/// block is never rewritten.
189///
190/// Each append occupies whole blocks: the bytes from the end of the data to
191/// the next block boundary are zero and are not caller data. Pack small
192/// records into one append (or use a batch) to avoid the padding.
193pub struct AppendRegion<P: Platform> {
194 store: Store<P>,
195 state: Arc<LiveRegion>,
196}
197
198impl<P: Platform> AppendRegion<P> {
199 pub(crate) fn from_parts(store: Store<P>, state: Arc<LiveRegion>) -> Self {
200 Self { store, state }
201 }
202}
203
204impl<P: Platform> Clone for AppendRegion<P> {
205 fn clone(&self) -> Self {
206 Self {
207 store: self.store.clone(),
208 state: Arc::clone(&self.state),
209 }
210 }
211}
212
213/// The tail of every prefix sync (`sync_through`, a set barrier's append
214/// tickets, `settle::durable_receipt`): publishes `snapshot`, the completed
215/// prefix read **before** the flush's ticket `need` was taken (a prefix read
216/// after it may hold bytes the flush does not cover), as synced when a data
217/// flush backs it, else durable, and builds its prefix receipt.
218pub(crate) fn sync_tail<P: Platform>(
219 store: &Store<P>,
220 state: &LiveRegion,
221 snapshot: u64,
222 need: u64,
223 synced: bool,
224) -> DurableReceipt
225where
226 P::Queue: Send,
227{
228 if let Some(f) = state.frontier.as_ref() {
229 if synced {
230 f.publish_synced(snapshot);
231 } else {
232 f.publish_durable(snapshot);
233 }
234 }
235 store.receipt(state, 0, snapshot, need, true, synced)
236}
237impl<P: Platform> AppendRegion<P>
238where
239 P::Queue: Send,
240{
241 /// Threads waiting on the region's completed prefix (an append waiting
242 /// for room in the window, a sync waiting for its appends). For tests.
243 #[doc(hidden)]
244 #[must_use]
245 pub fn waiting_on_appends(&self) -> u64 {
246 self.state.frontier.as_ref().map_or(0, |f| f.waiting())
247 }
248
249 fn frontier(&self) -> Result<&crate::frontier::AppendFrontier, Error> {
250 self.state
251 .frontier
252 .as_ref()
253 .ok_or_else(|| not_written(NotWrittenCause::NotReady, ctx(&self.state)))
254 }
255
256 /// The checks every append makes before touching the device, and the
257 /// pass that keeps a recycle or release out until the append is done.
258 pub(crate) fn writable(&self) -> Result<(&crate::frontier::AppendFrontier, Pass<'_>), Error> {
259 let pass = self.pass()?;
260 self.store.ready_for(&self.state)?;
261 let f = self.frontier()?;
262 if !self.state.positioned.load(Ordering::Acquire) {
263 return Err(not_written(
264 NotWrittenCause::NotPositioned,
265 ctx(&self.state),
266 ));
267 }
268 Ok((f, pass))
269 }
270
271 fn pass(&self) -> Result<Pass<'_>, Error> {
272 self.state
273 .gate
274 .enter()
275 .map_err(|why| retired_error(why, &self.state))
276 }
277
278 /// The generation this handle writes and reads. Recycling starts the
279 /// next one; embed it in your records to tell generations apart after a
280 /// crash.
281 #[must_use]
282 pub fn generation(&self) -> Generation {
283 self.state.generation
284 }
285
286 /// Starts a new generation: waits for every operation in flight on this
287 /// region, rewrites the region header with the next generation, makes it
288 /// durable, and returns a handle to the new generation with its append
289 /// position at 0. No other I/O: the old bytes stay on disk, so records
290 /// must carry their generation. Every handle, ticket and position of the
291 /// old generation is refused from then on (`StaleGeneration`).
292 ///
293 /// Must not be called from inside this region's own scan visitor, nor by
294 /// a thread whose own [`crate::Submitter`] has operations on this region
295 /// it has not reaped: recycle waits for them, and only that thread's
296 /// `reap` or `wait` finishes them (the same holds for `release`).
297 ///
298 /// # Errors
299 ///
300 /// `NotWritten(StaleGeneration | NotReady)` if the region was already
301 /// recycled or released; `NotWritten(ReadOnly)` if the store or the
302 /// region is read-only; `Io` if the store's queue could not be prepared
303 /// (nothing was written; the region stays usable); `Poisoned`;
304 /// `DurabilityUnknown`.
305 pub fn recycle(&self) -> Result<Self, Error> {
306 let state = self.store.recycle_region(&self.state)?;
307 Ok(Self {
308 store: self.store.clone(),
309 state,
310 })
311 }
312
313 /// Releases the region: waits for every operation in flight, records
314 /// the release in the region table (durable: from then on the release
315 /// stands), marks its header released (durable), and gives its space
316 /// back to the file system. Its contents are undefined from then on;
317 /// every handle to it is refused (`NotReady`), and its space is reused
318 /// by later provisioning. A header write that fails before it reaches
319 /// the device (refused, a queue that could not be prepared, a header
320 /// pair that would not encode or plan) still returns success: the space
321 /// is free for provisioning, but not given back to the file system. Not
322 /// from a thread with unreaped [`crate::Submitter`] operations on it
323 /// (see [`Self::recycle`]).
324 ///
325 /// # Errors
326 ///
327 /// `NotWritten(StaleGeneration | NotReady)` if the region was already
328 /// recycled or released; `NotWritten(ReadOnly)` if the store or the
329 /// region is read-only; on format 2, `NotWritten(Frozen)` while a table
330 /// page is stale or lost, and `NotWritten(Overlap)` when removing the
331 /// entry would change another region's verdict at the next open;
332 /// `Poisoned`; `DurabilityUnknown`; or `Io` if the space could not be
333 /// deallocated (the region is released either way). Once the table
334 /// records the release, a release fails only on a poisoned store or
335 /// with that `Io`.
336 pub fn release(self) -> Result<(), Error> {
337 self.store.release_region(&self.state)
338 }
339
340 pub(crate) fn store(&self) -> &Store<P> {
341 &self.store
342 }
343
344 pub(crate) fn state(&self) -> &Arc<LiveRegion> {
345 &self.state
346 }
347
348 /// Appends `data` (any length of at least one byte) and returns a ticket
349 /// once the write has completed. The data is not yet durable: pass the
350 /// ticket to [`Self::sync_through`], or use [`Self::append_durable`].
351 ///
352 /// The append starts at a block boundary chosen by store-io and is
353 /// zero-padded to the next one. Data larger than the largest pooled
354 /// buffer is written in several pieces, kept in flight together.
355 ///
356 /// # Errors
357 ///
358 /// `NotWritten` (empty data, region full, pool exhausted, read-only, not
359 /// positioned), `Io` if the store's queue could not be prepared (nothing
360 /// was reserved or written), `Poisoned`, or `DurabilityUnknown` (the
361 /// domain is now poisoned).
362 pub fn append(&self, data: &[u8]) -> Result<WriteTicket, Error> {
363 self.append_with(data, || Ok(()), |()| {})
364 }
365
366 /// [`Self::append`], running `before_reserve` once its lane is held and
367 /// before its reservation (again on each retry for room), and handing
368 /// what it returned to `on_reserved` once the space is reserved, before
369 /// the write is submitted (a dependent write's gate passes, `after.rs`).
370 pub(crate) fn append_with<T>(
371 &self,
372 data: &[u8],
373 before_reserve: impl FnMut() -> Result<T, Error>,
374 on_reserved: impl FnOnce(T),
375 ) -> Result<WriteTicket, Error> {
376 let s = &self.store;
377 let (f, _pass) = self.writable()?;
378 let c = ctx(&self.state);
379 if data.is_empty() {
380 return Err(not_written(NotWrittenCause::Empty, c));
381 }
382 // Everything that can fail without touching the device happens first:
383 // the first piece is staged before any space is reserved.
384 let mut stream = s.stage(data, c)?;
385 let (mut lane, reserved, before) =
386 s.lane_and_reservation_with(&self.state, f, data.len() as u64, c, before_reserve)?;
387 on_reserved(before);
388 let r = reserved.range();
389 stream.at(r.start);
390 // Until the pump returns the guard poisons on any exit (a panic
391 // included); then the settle step completes the reservation, or
392 // poisons: past it every failure leaves a hole.
393 let outcome = s.pump(
394 &self.state,
395 &mut lane,
396 &mut Mapped::new(&self.state.map, &mut stream),
397 );
398 let r = reserved.disarm();
399 settle_pumped(s, &self.state, Wrote::Append(r), outcome)
400 }
401
402 /// Makes everything appended up to and including `ticket` durable.
403 ///
404 /// # Errors
405 ///
406 /// `NotWritten(ForeignPosition | StaleGeneration)` for a ticket of another
407 /// region or generation; `Poisoned`; `DurabilityUnknown` if the flush
408 /// failed.
409 pub fn sync_through(&self, ticket: &WriteTicket) -> Result<DurableReceipt, Error> {
410 self.sync_through_from(ticket, None)
411 }
412
413 /// [`Self::sync_through`] for a commit that entered at `entry`
414 /// (`Store::write_entry`, read before its write).
415 pub(crate) fn sync_through_from(
416 &self,
417 ticket: &WriteTicket,
418 entry: Option<u64>,
419 ) -> Result<DurableReceipt, Error> {
420 let s = &self.store;
421 let f = self.frontier()?;
422 let c = ctx(&self.state);
423 self.state
424 .gate
425 .check()
426 .map_err(|why| retired_error(why, &self.state))?;
427 if ticket.epoch != s.inner.epoch
428 || ticket.volume != s.inner.volume
429 || ticket.region != self.state.id
430 {
431 return Err(not_written(NotWrittenCause::ForeignPosition, c));
432 }
433 if ticket.generation != self.state.generation {
434 return Err(not_written(NotWrittenCause::StaleGeneration, c));
435 }
436 // Already durable: no barrier. A forced region on a power-safe store
437 // counts only what a data flush made durable (D-102).
438 let flushes = s.inner.policy.flush;
439 let synced = flushes || self.state.forced();
440 let durable = if synced && !flushes {
441 f.synced()
442 } else {
443 f.durable()
444 };
445 if durable >= ticket.end {
446 s.inner.domain.count_already_durable();
447 return Ok(s.receipt(&self.state, 0, durable, 0, true, synced));
448 }
449 let domain = &s.inner.domain;
450 if !f.wait_completed(ticket.end, || domain.poisoned().is_some()) {
451 return Err(Error::Poisoned {
452 first: domain.poisoned().unwrap_or(FirstCause {
453 op: None,
454 raw: None,
455 }),
456 });
457 }
458 // Snapshot the completed prefix BEFORE the barrier takes its ticket.
459 let through = f.completed();
460 let (need, synced) = s.barrier_for(&self.state, entry)?;
461 Ok(sync_tail(s, &self.state, through, need, synced))
462 }
463
464 /// Appends `data` and makes it durable before returning.
465 ///
466 /// # Errors
467 ///
468 /// As [`Self::append`] and [`Self::sync_through`].
469 pub fn append_durable(&self, data: &[u8]) -> Result<(RegionPos, DurableReceipt), Error> {
470 // Timed for auto-tuning from before the write (D-160).
471 let entry = self.store.write_entry(&self.state);
472 let t = self.append(data)?;
473 let pos = t.pos();
474 let r = self.sync_through_from(&t, entry)?;
475 Ok((pos, r))
476 }
477
478 /// A batch: many records packed back to back, written together and made
479 /// durable with one barrier. See [`AppendBatch`].
480 #[must_use]
481 pub fn batch(&self) -> AppendBatch<P> {
482 AppendBatch::new(self.clone())
483 }
484
485 /// Reads raw bytes at a block-aligned offset into `out` (any length).
486 /// Returns the bytes read, fewer than `out.len()` only at the end of the
487 /// region's data area.
488 ///
489 /// The bytes are read into store-io's own aligned buffers and copied
490 /// into `out`: direct I/O needs aligned memory, and a read whose wait
491 /// fails stays in flight, so it must never target the caller's memory.
492 /// To read with no copy, use [`Self::scan`] (it lends store-io's
493 /// buffers) or a [`crate::Submitter`]'s `read_append` (it reads into a
494 /// buffer taken from the store).
495 ///
496 /// # Errors
497 ///
498 /// `NotWritten` for a misaligned or out-of-range read;
499 /// `Corruption(MediaError)` if the device reports an unreadable range,
500 /// with the exact range in the error's context; `Io` with the raw code
501 /// for any other failure.
502 pub fn read(&self, offset: u64, out: &mut [u8]) -> Result<usize, Error> {
503 let _pass = self.pass()?;
504 self.store.read_at(&self.state, offset, out)
505 }
506
507 /// Reads the region in order from the block-aligned offset `from` to the
508 /// end of its data area, handing every piece to `visit`; return
509 /// [`ControlFlow::Break`] to stop (typically where your valid data ends).
510 ///
511 /// The scan reads with direct I/O and several large reads in flight. It
512 /// never stops at bad data: a range the device cannot read is narrowed
513 /// to whole sectors (the device's logical blocks where the probe
514 /// reported the direct-I/O alignment, else store blocks), reported as
515 /// [`ScanItem::Unreadable`], and skipped.
516 /// Scans work on read-only and poisoned stores.
517 ///
518 /// `visit` may call the store (read, write, provision): it never runs
519 /// while the scan holds a queue another call waits for (on a shared
520 /// queue the scan puts it back before delivering). This region's
521 /// `recycle` and `release` wait for the scan itself to end, so they
522 /// must not be called from `visit`; nor may an append wait there for
523 /// room that only the same thread's [`crate::Submitter`] can free.
524 ///
525 /// # Errors
526 ///
527 /// `NotWritten(Misaligned | OutOfBounds | PoolExhausted)` before any I/O;
528 /// `Io` if a read fails for a reason other than the media.
529 pub fn scan(
530 &self,
531 from: u64,
532 visit: impl FnMut(ScanItem<'_>) -> ControlFlow<()>,
533 ) -> Result<ScanSummary, Error> {
534 let _pass = self.pass()?;
535 self.store
536 .scan_reported(&self.state, from, self.state.map.len(), visit)
537 }
538
539 /// Moves the append position forward to `offset` (rounded up to a block),
540 /// typically after recovery found where valid data ends. Never moves
541 /// backwards, and only while no append is in flight.
542 ///
543 /// The skipped range counts as durable: a writable open flushed the
544 /// device before reading anything, so whatever recovery found is on
545 /// media. Bytes appended in this open and not yet made durable never do:
546 /// then the durable point stays where it is, and the next sync that
547 /// flushes covers both (`durable_through` reads lower until then).
548 /// A read-only open flushes nothing, so there `resume_at` is refused
549 /// (D-072); appends are refused there anyway, and so is a poisoned
550 /// store.
551 ///
552 /// # Errors
553 ///
554 /// `NotWritten(ReadOnly | TooManyInFlight | OutOfBounds)`, `Poisoned`.
555 pub fn resume_at(&self, offset: u64) -> Result<u64, Error> {
556 if self.store.inner.read_only || self.state.is_read_only() {
557 return Err(Error::NotWritten {
558 cause: NotWrittenCause::ReadOnly,
559 ctx: ctx(&self.state),
560 });
561 }
562 // A poison landing after this check is harmless: a failed append
563 // leaves the prefix below the tail (refused below), a failed flush
564 // leaves the durable point below it (not raised).
565 if let Some(first) = self.store.inner.domain.poisoned() {
566 return Err(Error::Poisoned { first });
567 }
568 let _pass = self.pass()?;
569 let at = self
570 .frontier()?
571 .resume_at(offset)
572 .map_err(|cause| Error::NotWritten {
573 cause,
574 ctx: ctx(&self.state),
575 })?;
576 // Before the region reads as positioned: a dependent write that sees
577 // it positioned sees this (`after.rs`).
578 let inner = &self.store.inner;
579 crate::frontier::record_resume(
580 (inner.volume, self.state.id, self.state.generation),
581 inner.epoch,
582 at,
583 );
584 self.state
585 .positioned
586 .store(true, std::sync::atomic::Ordering::Release);
587 Ok(at)
588 }
589
590 /// Where the next append will start.
591 #[must_use]
592 pub fn tail(&self) -> u64 {
593 self.state
594 .frontier
595 .as_ref()
596 .map_or(0, crate::frontier::AppendFrontier::tail)
597 }
598
599 /// The bytes of `range` (region offsets, any start and end) as a
600 /// [`ReadStream`](crate::ReadStream) of owned chunks in order, with the
601 /// default [`StreamOptions`](crate::StreamOptions): a background
602 /// stream. Only the durable prefix as of the call is read (`range` is
603 /// clamped to [`Self::durable_through`]; empty: an empty stream).
604 ///
605 /// # Errors
606 ///
607 /// `NotWritten`: `OutOfBounds` (`range` reversed or past [`Self::len`]),
608 /// `NotPositioned` (after a reopen, before `resume_at`: so never on a
609 /// read-only open, which refuses `resume_at`; scan it instead),
610 /// `StaleGeneration` or `NotReady` (recycled or released),
611 /// `PoolExhausted` (too few buffers free under the pool's background
612 /// cap), `InvalidOptions`; `Io` if its queue could not be made.
613 pub fn read_stream(&self, range: std::ops::Range<u64>) -> Result<crate::ReadStream<P>, Error> {
614 self.read_stream_with(range, crate::StreamOptions::default())
615 }
616
617 /// [`Self::read_stream`] with `opts`.
618 ///
619 /// # Errors
620 ///
621 /// As [`Self::read_stream`].
622 pub fn read_stream_with(
623 &self,
624 range: std::ops::Range<u64>,
625 opts: crate::StreamOptions,
626 ) -> Result<crate::ReadStream<P>, Error> {
627 let sr = self.stream_range(range)?;
628 self.store.prefetch(&[sr], opts)
629 }
630
631 /// `range` as one range for [`Store::prefetch`](crate::Store::prefetch),
632 /// checked now as [`Self::read_stream`] checks it (and clamped to the
633 /// durable prefix as of this call).
634 ///
635 /// # Errors
636 ///
637 /// As [`Self::read_stream`]: `OutOfBounds`, `NotPositioned`,
638 /// `StaleGeneration`, `NotReady`.
639 pub fn stream_range(
640 &self,
641 range: std::ops::Range<u64>,
642 ) -> Result<crate::StreamRange<'_, P>, Error> {
643 self.state
644 .gate
645 .live()
646 .map_err(|why| retired_error(why, &self.state))?;
647 if !self.state.positioned.load(Ordering::Acquire) {
648 return Err(not_written(
649 NotWrittenCause::NotPositioned,
650 ctx(&self.state),
651 ));
652 }
653 if range.start > range.end || range.end > self.state.map.len() {
654 return Err(not_written(
655 NotWrittenCause::OutOfBounds,
656 crate::io::ctx_range(&self.state, range.start, range.end),
657 ));
658 }
659 // The durable prefix as of the call (R5.5).
660 let end = range.end.min(self.durable_through());
661 let start = range.start.min(end);
662 Ok(crate::StreamRange::new(
663 &self.store,
664 &self.state,
665 start..end,
666 ))
667 }
668
669 /// Every byte before this offset is durable.
670 #[must_use]
671 pub fn durable_through(&self) -> u64 {
672 self.state
673 .frontier
674 .as_ref()
675 .map_or(0, crate::frontier::AppendFrontier::durable)
676 }
677
678 /// Data-area size in bytes.
679 #[must_use]
680 pub fn len(&self) -> u64 {
681 self.state.map.len()
682 }
683
684 /// How far the data area can grow without new space: the region's
685 /// capacity in bytes (the sum of its extents).
686 #[must_use]
687 pub fn capacity(&self) -> u64 {
688 self.state.map.capacity()
689 }
690
691 /// Grows the data area to at least `new_size` bytes (rounded up to the
692 /// block size): the new range is filled, flushed and verified before
693 /// one table page write commits the new size (spec §12). Within the
694 /// capacity nothing else moves; past it the region gains one extent of
695 /// at least 1 MiB and an eighth of its capacity (after its last one
696 /// when that space is free), listed in its header before the commit.
697 /// Returns the new size; on a writable store a size at most the current
698 /// one writes nothing and returns it. Positions and tickets taken
699 /// before stay valid.
700 ///
701 /// # Errors
702 ///
703 /// `Unsupported` in container format 1; `NotWritten` (`TooLarge` past
704 /// 2^32 blocks; `ReadOnly`; `Frozen` while a table page is stale or
705 /// lost; `NotReady` if the filled range does not verify;
706 /// `StaleGeneration` for a recycled region; `QueueFull`); `NoSpace`
707 /// (`RegionLimit` for a capped region, `Extents` when its header lists
708 /// as many extents as it can hold, `Caller` past the tag's cap,
709 /// `Growth` when the device runs out); `Io` if the range could not be
710 /// probed or allocated; `Corruption` for metadata that does not decode;
711 /// `Poisoned`; `DurabilityUnknown` (a fill, a flush, the header or the
712 /// page write failed after submission: poisoned).
713 pub fn grow(&self, new_size: u64) -> Result<u64, Error> {
714 let _pass = self.pass()?;
715 self.store.grow_region2(&self.state, new_size)
716 }
717
718 /// Whether the data area is empty (never for a provisioned region).
719 #[must_use]
720 pub fn is_empty(&self) -> bool {
721 self.state.map.len() == 0
722 }
723
724 /// Block size: appends are padded to it and reads must be aligned to it.
725 #[must_use]
726 pub fn block_size(&self) -> u64 {
727 self.store.inner.layout.block()
728 }
729}
730
731// ---------------------------------------------------------------------------
732// Page regions
733// ---------------------------------------------------------------------------
734
735/// A region of caller-addressed blocks, overwritten in place.
736pub struct PageRegion<P: Platform> {
737 store: Store<P>,
738 state: Arc<LiveRegion>,
739}
740
741impl<P: Platform> PageRegion<P> {
742 pub(crate) fn from_parts(store: Store<P>, state: Arc<LiveRegion>) -> Self {
743 Self { store, state }
744 }
745}
746
747impl<P: Platform> Clone for PageRegion<P> {
748 fn clone(&self) -> Self {
749 Self {
750 store: self.store.clone(),
751 state: Arc::clone(&self.state),
752 }
753 }
754}
755
756impl<P: Platform> PageRegion<P>
757where
758 P::Queue: Send,
759{
760 /// A checked position: `offset` must be block-aligned and inside the
761 /// region.
762 ///
763 /// # Errors
764 ///
765 /// `NotWritten(Misaligned | OutOfBounds)`.
766 pub fn pos(&self, offset: u64) -> Result<RegionPos, Error> {
767 let c = ctx(&self.state);
768 if offset % self.store.block() != 0 {
769 return Err(not_written(NotWrittenCause::Misaligned, c));
770 }
771 if offset >= self.state.map.len() {
772 return Err(not_written(NotWrittenCause::OutOfBounds, c));
773 }
774 Ok(RegionPos {
775 volume: self.store.inner.volume,
776 region: self.state.id,
777 generation: self.state.generation,
778 offset,
779 })
780 }
781
782 /// Checks a write of `len` bytes at `pos`; returns its end.
783 pub(crate) fn check(&self, pos: &RegionPos, len: usize) -> Result<u64, Error> {
784 let c = ctx(&self.state);
785 if pos.volume != self.store.inner.volume || pos.region != self.state.id {
786 return Err(not_written(NotWrittenCause::ForeignPosition, c));
787 }
788 if pos.generation != self.state.generation {
789 return Err(not_written(NotWrittenCause::StaleGeneration, c));
790 }
791 if len == 0 {
792 return Err(not_written(NotWrittenCause::Empty, c));
793 }
794 if (len as u64) % self.store.block() != 0 {
795 return Err(not_written(NotWrittenCause::Misaligned, c));
796 }
797 match pos.offset.checked_add(len as u64) {
798 Some(end) if end <= self.state.map.len() => Ok(end),
799 _ => Err(not_written(NotWrittenCause::OutOfBounds, c)),
800 }
801 }
802
803 pub(crate) fn store(&self) -> &Store<P> {
804 &self.store
805 }
806
807 pub(crate) fn state(&self) -> &Arc<LiveRegion> {
808 &self.state
809 }
810
811 pub(crate) fn pass(&self) -> Result<Pass<'_>, Error> {
812 self.state
813 .gate
814 .enter()
815 .map_err(|why| retired_error(why, &self.state))
816 }
817
818 /// The generation this handle writes and reads.
819 #[must_use]
820 pub fn generation(&self) -> Generation {
821 self.state.generation
822 }
823
824 /// Starts a new generation (see [`AppendRegion::recycle`]); positions and
825 /// tickets of the old generation are refused from then on.
826 ///
827 /// # Errors
828 ///
829 /// As [`AppendRegion::recycle`].
830 pub fn recycle(&self) -> Result<Self, Error> {
831 let state = self.store.recycle_region(&self.state)?;
832 Ok(Self {
833 store: self.store.clone(),
834 state,
835 })
836 }
837
838 /// Releases the region (see [`AppendRegion::release`]).
839 ///
840 /// # Errors
841 ///
842 /// As [`AppendRegion::release`].
843 pub fn release(self) -> Result<(), Error> {
844 self.store.release_region(&self.state)
845 }
846
847 /// Writes whole blocks at `pos` and returns a ticket once the write has
848 /// completed. `data.len()` must be a multiple of [`Self::block_size`];
849 /// any size is accepted (large writes go out in several pieces).
850 ///
851 /// # Errors
852 ///
853 /// `NotWritten` (foreign, stale, misaligned, out of range, pool
854 /// exhausted), `Io` (refused by the platform before reaching the device;
855 /// nothing is poisoned), `Poisoned`, or `DurabilityUnknown`.
856 pub fn write(&self, pos: RegionPos, data: &[u8]) -> Result<WriteTicket, Error> {
857 self.write_with(pos, data, || Ok(()))
858 }
859
860 /// [`Self::write`], running `before_issue` once its lane is held and
861 /// dropping what it returned before the write is issued (a dependent
862 /// write's gate passes, `after.rs`).
863 pub(crate) fn write_with<T>(
864 &self,
865 pos: RegionPos,
866 data: &[u8],
867 before_issue: impl FnMut() -> Result<T, Error>,
868 ) -> Result<WriteTicket, Error> {
869 let s = &self.store;
870 let _pass = self.pass()?;
871 s.ready_for(&self.state)?;
872 let end = self.check(&pos, data.len())?;
873 let c = ctx_range(&self.state, pos.offset, end);
874 let outcome = s.write_staged_with(&self.state, pos.offset, data, c, before_issue)?;
875 let wrote = Wrote::Page {
876 start: pos.offset,
877 end,
878 };
879 settle_pumped(s, &self.state, wrote, outcome)
880 }
881
882 /// Makes the ticket's write durable, together with every other write to
883 /// this store that completed before the call.
884 ///
885 /// # Errors
886 ///
887 /// `NotWritten(ForeignPosition | StaleGeneration)`, `Poisoned`, or
888 /// `DurabilityUnknown`.
889 pub fn sync_through(&self, ticket: &WriteTicket) -> Result<DurableReceipt, Error> {
890 self.sync_through_from(ticket, None)
891 }
892
893 /// [`Self::sync_through`] for a commit that entered at `entry`
894 /// (`Store::write_entry`, read before its write).
895 fn sync_through_from(
896 &self,
897 ticket: &WriteTicket,
898 entry: Option<u64>,
899 ) -> Result<DurableReceipt, Error> {
900 let s = &self.store;
901 let c = ctx(&self.state);
902 if ticket.epoch != s.inner.epoch
903 || ticket.volume != s.inner.volume
904 || ticket.region != self.state.id
905 {
906 return Err(not_written(NotWrittenCause::ForeignPosition, c));
907 }
908 if ticket.generation != self.state.generation {
909 return Err(not_written(NotWrittenCause::StaleGeneration, c));
910 }
911 self.state
912 .gate
913 .check()
914 .map_err(|why| retired_error(why, &self.state))?;
915 let (need, synced) = s.barrier_for(&self.state, entry)?;
916 Ok(s.receipt(&self.state, ticket.start, ticket.end, need, false, synced))
917 }
918
919 /// Writes whole blocks at `pos` and makes them durable before returning.
920 ///
921 /// # Errors
922 ///
923 /// As [`Self::write`] and [`Self::sync_through`].
924 pub fn write_durable(&self, pos: RegionPos, data: &[u8]) -> Result<DurableReceipt, Error> {
925 // Timed for auto-tuning from before the write (D-160).
926 let entry = self.store.write_entry(&self.state);
927 let t = self.write(pos, data)?;
928 self.sync_through_from(&t, entry)
929 }
930
931 /// A batch: many page writes submitted together and made durable with
932 /// one barrier. See [`PageBatch`].
933 #[must_use]
934 pub fn batch(&self) -> PageBatch<P> {
935 PageBatch::new(self.clone())
936 }
937
938 /// Reads at `pos` into `out` (any length; reads whole blocks internally).
939 /// Returns the bytes read, fewer than `out.len()` only at the end of the
940 /// region.
941 ///
942 /// The bytes are read into store-io's own aligned buffers and copied
943 /// into `out`: direct I/O needs aligned memory, and a read whose wait
944 /// fails stays in flight, so it must never target the caller's memory.
945 /// To read with no copy, use [`Self::scan`] (it lends store-io's
946 /// buffers) or a [`crate::Submitter`]'s `read_page` (it reads into a
947 /// buffer taken from the store).
948 ///
949 /// # Errors
950 ///
951 /// `NotWritten` for a foreign, stale or out-of-range position;
952 /// `Corruption(MediaError)` for an unreadable range, with the exact range
953 /// in the error's context; `Io` with the raw code for any other failure.
954 pub fn read(&self, pos: RegionPos, out: &mut [u8]) -> Result<usize, Error> {
955 let c = ctx(&self.state);
956 if pos.volume != self.store.inner.volume || pos.region != self.state.id {
957 return Err(not_written(NotWrittenCause::ForeignPosition, c));
958 }
959 if pos.generation != self.state.generation {
960 return Err(not_written(NotWrittenCause::StaleGeneration, c));
961 }
962 let _pass = self.pass()?;
963 self.store.read_at(&self.state, pos.offset, out)
964 }
965
966 /// Reads the region in order from the block-aligned offset `from` to the
967 /// end of its data area, handing every piece to `visit`; return
968 /// [`ControlFlow::Break`] to stop (typically where your valid data ends).
969 ///
970 /// The scan reads with direct I/O and several large reads in flight. It
971 /// never stops at bad data: a range the device cannot read is narrowed
972 /// to whole sectors (the device's logical blocks where the probe
973 /// reported the direct-I/O alignment, else store blocks), reported as
974 /// [`ScanItem::Unreadable`], and skipped.
975 /// Scans work on read-only and poisoned stores.
976 ///
977 /// `visit` may call the store (read, write, provision): it never runs
978 /// while the scan holds a queue another call waits for (on a shared
979 /// queue the scan puts it back before delivering). This region's
980 /// `recycle` and `release` wait for the scan itself to end, so they
981 /// must not be called from `visit`; nor may an append wait there for
982 /// room that only the same thread's [`crate::Submitter`] can free.
983 ///
984 /// # Errors
985 ///
986 /// `NotWritten(Misaligned | OutOfBounds | PoolExhausted)` before any I/O;
987 /// `Io` if a read fails for a reason other than the media.
988 pub fn scan(
989 &self,
990 from: u64,
991 visit: impl FnMut(ScanItem<'_>) -> ControlFlow<()>,
992 ) -> Result<ScanSummary, Error> {
993 let _pass = self.pass()?;
994 self.store
995 .scan_reported(&self.state, from, self.state.map.len(), visit)
996 }
997
998 /// The bytes of `range` (region offsets, any start and end) as a
999 /// [`ReadStream`](crate::ReadStream) of owned chunks in order, with the
1000 /// default [`StreamOptions`](crate::StreamOptions): a background stream.
1001 /// Not a snapshot: a write made while streaming may show in it.
1002 ///
1003 /// # Errors
1004 ///
1005 /// `NotWritten`: `OutOfBounds` (`range` reversed or past [`Self::len`]),
1006 /// `StaleGeneration` or `NotReady` (recycled or released),
1007 /// `PoolExhausted` (too few buffers free under the pool's background
1008 /// cap), `InvalidOptions`; `Io` if its queue could not be made.
1009 pub fn read_stream(&self, range: std::ops::Range<u64>) -> Result<crate::ReadStream<P>, Error> {
1010 self.read_stream_with(range, crate::StreamOptions::default())
1011 }
1012
1013 /// [`Self::read_stream`] with `opts`.
1014 ///
1015 /// # Errors
1016 ///
1017 /// As [`Self::read_stream`].
1018 pub fn read_stream_with(
1019 &self,
1020 range: std::ops::Range<u64>,
1021 opts: crate::StreamOptions,
1022 ) -> Result<crate::ReadStream<P>, Error> {
1023 let sr = self.stream_range(range)?;
1024 self.store.prefetch(&[sr], opts)
1025 }
1026
1027 /// `range` as one range for [`Store::prefetch`](crate::Store::prefetch),
1028 /// checked now as [`Self::read_stream`] checks it.
1029 ///
1030 /// # Errors
1031 ///
1032 /// As [`Self::read_stream`]: `OutOfBounds`, `StaleGeneration`,
1033 /// `NotReady`.
1034 pub fn stream_range(
1035 &self,
1036 range: std::ops::Range<u64>,
1037 ) -> Result<crate::StreamRange<'_, P>, Error> {
1038 self.state
1039 .gate
1040 .live()
1041 .map_err(|why| retired_error(why, &self.state))?;
1042 if range.start > range.end || range.end > self.state.map.len() {
1043 return Err(not_written(
1044 NotWrittenCause::OutOfBounds,
1045 crate::io::ctx_range(&self.state, range.start, range.end),
1046 ));
1047 }
1048 Ok(crate::StreamRange::new(&self.store, &self.state, range))
1049 }
1050
1051 /// Block size: writes must be whole blocks.
1052 #[must_use]
1053 pub fn block_size(&self) -> u64 {
1054 self.store.inner.layout.block()
1055 }
1056
1057 /// Data-area size in bytes.
1058 #[must_use]
1059 pub fn len(&self) -> u64 {
1060 self.state.map.len()
1061 }
1062
1063 /// How far the data area can grow without new space: the region's
1064 /// capacity in bytes (the sum of its extents).
1065 #[must_use]
1066 pub fn capacity(&self) -> u64 {
1067 self.state.map.capacity()
1068 }
1069
1070 /// Grows the data area to at least `new_size` bytes (rounded up to the
1071 /// block size): the new range is filled, flushed and verified before
1072 /// one table page write commits the new size (spec §12). Within the
1073 /// capacity nothing else moves; past it the region gains one extent of
1074 /// at least 1 MiB and an eighth of its capacity (after its last one
1075 /// when that space is free), listed in its header before the commit.
1076 /// Returns the new size; on a writable store a size at most the current
1077 /// one writes nothing and returns it. Positions and tickets taken
1078 /// before stay valid.
1079 ///
1080 /// # Errors
1081 ///
1082 /// `Unsupported` in container format 1; `NotWritten` (`TooLarge` past
1083 /// 2^32 blocks; `ReadOnly`; `Frozen` while a table page is stale or
1084 /// lost; `NotReady` if the filled range does not verify;
1085 /// `StaleGeneration` for a recycled region; `QueueFull`); `NoSpace`
1086 /// (`RegionLimit` for a capped region, `Extents` when its header lists
1087 /// as many extents as it can hold, `Caller` past the tag's cap,
1088 /// `Growth` when the device runs out); `Io` if the range could not be
1089 /// probed or allocated; `Corruption` for metadata that does not decode;
1090 /// `Poisoned`; `DurabilityUnknown` (a fill, a flush, the header or the
1091 /// page write failed after submission: poisoned).
1092 pub fn grow(&self, new_size: u64) -> Result<u64, Error> {
1093 let _pass = self.pass()?;
1094 self.store.grow_region2(&self.state, new_size)
1095 }
1096
1097 /// Whether the data area is empty (never for a provisioned region).
1098 #[must_use]
1099 pub fn is_empty(&self) -> bool {
1100 self.state.map.len() == 0
1101 }
1102}
1103
1104impl core::fmt::Debug for LiveRegion {
1105 fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
1106 f.debug_struct("Region")
1107 .field("name", &self.name.as_str())
1108 .field("id", &self.id)
1109 .field("kind", &self.kind)
1110 .field("generation", &self.generation)
1111 .field("size", &self.map.len())
1112 .finish()
1113 }
1114}
1115
1116impl<P: Platform> core::fmt::Debug for AppendRegion<P> {
1117 fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
1118 f.debug_tuple("AppendRegion").field(&*self.state).finish()
1119 }
1120}
1121
1122impl<P: Platform> core::fmt::Debug for PageRegion<P> {
1123 fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
1124 f.debug_tuple("PageRegion").field(&*self.state).finish()
1125 }
1126}