Skip to main content

store_io_engine/
store.rs

1//! [`Store`]: one container on one device.
2//!
3//! A store owns the container file, the ownership lock, the device's flush
4//! domain, the buffer pool and the queues, and the in-memory copy of the
5//! volume record and region table. Region handles share it through an `Arc`.
6//!
7//! The metadata lock (`meta`) is taken only by lifecycle operations
8//! (provision, release, lookup by name); reads and writes of region data never
9//! take it.
10
11use std::sync::{Arc, Mutex, MutexGuard, PoisonError};
12
13pub use crate::v2::{DamagedSlot, Healed, MetadataReport, SlotDamage};
14
15use store_io_buf::{BufPool, PoolConfig};
16use store_io_core::class::{DurabilityClass, ReceiptLabel};
17use store_io_core::decide::{ClassDecision, DurableOpen, Trust};
18use store_io_core::errno::{Stage, classify};
19use store_io_core::error::{
20    Error, ErrorContext, FirstCause, Named, NotWrittenCause, Op, OsError, ReserveTag,
21};
22use store_io_core::evidence::Evidence;
23use store_io_core::id::{Generation, RegionId, VolumeId};
24use store_io_core::observe::{Observer, Snapshot};
25use store_io_format::meta::{RegionKind, RegionName, TableEntry, VolumeMeta};
26use store_io_platform::{FileName, IoOp, IoPath, LinuxTier, Platform, Queue, QueueConfig};
27
28use crate::domain::Domain;
29use crate::exec::{Lane, QueueSet, Ran, run};
30use crate::frontier::AppendFrontier;
31use crate::gate::Gate;
32use crate::layout::{Layout, TABLE_OBJECT, VOLUME_OBJECT};
33use crate::slots::{NextSlot, PairAt, PairRead, write_slot};
34use crate::space::Accounting;
35
36/// The container file name inside the store directory.
37pub const CONTAINER: &str = "store.sio";
38
39/// Explicit, reported choices for a store. Every durability-affecting field
40/// appears in [`Store::report`].
41#[derive(Debug, Clone)]
42pub struct StoreOptions {
43    /// Certificate, attestation and override inputs to the class decision.
44    pub trust: Trust,
45    /// Minimum block (slot) size in bytes; the device may require more.
46    pub block_size: Option<u32>,
47    /// Region-table slot size as a power of two (14 = 16 KiB: a format-2
48    /// table page of 169 regions; format 1's single table, 253).
49    pub log2_table_slot: u8,
50    /// Largest single I/O of the blocking layers, and the batch buffer size
51    /// (bytes; rounded up to a power of two, 128 KiB to 1 GiB).
52    pub max_io: usize,
53    /// Buffers per size class in the pool, up to `max_io` (at most
54    /// 16,777,215; more fails `create` and `open` with
55    /// `NotWritten(PoolExhausted)`).
56    pub buffers_per_class: u32,
57    /// Largest single write or read a `Submitter` takes, from one buffer, as
58    /// one OS write or read operation (bytes; rounded up to a power of two,
59    /// at most 1 GiB): the pool holds `large_buffers` buffers of that size,
60    /// used by submitters only (D-101). Ignored when not above `max_io`.
61    pub max_write: usize,
62    /// Buffers of `max_write` bytes in the pool (0: none, so a submitter's
63    /// largest operation is `max_io`; at most 16,777,215). The defaults add
64    /// 32 MiB on top of about 128 MiB of ordinary buffers per store: Windows
65    /// charges the whole pool to the commit limit at creation; on both
66    /// systems a buffer becomes resident when first used, and the most
67    /// recently returned buffer of a size is reused first, so only about as
68    /// many as are in use at once become resident. A pool too large to map
69    /// fails `create` and `open` with `NotWritten(PoolExhausted)`.
70    pub large_buffers: u32,
71    /// Queues shared by the blocking layers (0 = number of CPUs). On Linux
72    /// they perform their operations inline on either tier; the parallel
73    /// lanes, as many again and created on first use, are io_uring rings on
74    /// the io_uring tier, each holding two file descriptors (its ring and
75    /// its eventfd), as does each `Submitter`.
76    pub queues: usize,
77    /// Operations in flight per queue.
78    pub queue_depth: u32,
79    /// Worker threads for the platform's submit pool, which runs the
80    /// independent operations of a page batch or a large read concurrently
81    /// on the Linux synchronous tier (D-051, D-096); 0 = one per CPU, at most
82    /// 32. The pool is shared by every store on the platform and sized by the
83    /// first store that uses it (at most 256); other backends ignore this.
84    pub t3_pool_threads: u32,
85    /// How far, in bytes, a new append may start ahead of the oldest
86    /// incomplete one on the same region, in blocks: rounded down to a power
87    /// of two, or, on a region that cannot grow past a size smaller than the
88    /// window (format 1's regions, a capped format-2 region), that size
89    /// rounded up (so a region never limits its own appends); at least 2 and
90    /// at most 2^20 blocks. Each append region that has appended since the
91    /// open or its last recycle holds 8 bytes of memory per block of its
92    /// ring: a format-2 region that may grow (no `max_size`) holds the
93    /// window's whatever its size, 32 KiB at the default 16 MiB and 4 KiB
94    /// blocks (2 GiB for 65,536 such regions; a cap, or a smaller window,
95    /// bounds it). Default 16 MiB.
96    pub append_window: u64,
97    /// Fill new regions with the keyed incompressible pattern instead of
98    /// zeros (for thin, virtual, deduplicating or zero-detecting stacks).
99    pub keyed_fill: bool,
100    /// The kernel interface on Linux (D-082): `Auto` (io_uring where the
101    /// kernel accepts it), `Sync` or `Uring`. Read by the native API when
102    /// it makes the platform; the choice is in [`StoreReport::io`]. Other
103    /// systems ignore it.
104    pub linux_tier: LinuxTier,
105    /// Register the store's container file with each `Submitter`'s queue
106    /// where the backend can (D-078, D-107: on Linux's io_uring tier, a
107    /// read-only twin of it as a direct descriptor on the submitter's ring),
108    /// so its reads skip the per-operation file lookup in the kernel; writes
109    /// stay on the store's own descriptor, so a write in flight keeps the
110    /// store locked until it completes. Best effort: a
111    /// registration the kernel refuses is counted in `Store::io_path`, never
112    /// an error. On by default: the gates measured reads 1.03x to 1.08x of
113    /// an unregistered ring (G3d, CI); writes take the unregistered path.
114    /// Buffers
115    /// are never registered: reads into registered buffers hang forever on
116    /// some kernels (at least Linux 6.5.0 and 6.6.0), which no probe
117    /// detects (D-107).
118    pub register_file: bool,
119    /// Flushes the store runs at once (1 to 4; default 1, one in flight as
120    /// before, D-124). Several can overlap, meant for drives that execute
121    /// flushes in parallel (NVMe; not yet measured on one): each runs on a
122    /// handle and queue of its own, so each reports its own failures, and a
123    /// receipt still needs every flush up to its own to have succeeded.
124    /// Platforms that cannot keep flush failures per handle run one (Linux
125    /// 4.16 and later can); the number run is in
126    /// [`StoreReport::flushes_in_flight`]. Measure on the target drive
127    /// before raising it: at eight writers, 2 and 4 gained nothing on a
128    /// memory-backed device and lost on a virtual disk (1,314, 974 and 595
129    /// durable writes a second at 1, 2 and 4).
130    pub flushes_in_flight: u8,
131    /// Regions a format-2 store can hold, fixed at create (default 65,536:
132    /// a table of 12.1 MiB at 16 KiB table pages, written at create and
133    /// read at every open; smaller stores may ask for fewer; at most
134    /// 4,194,304). Ignored by format 1. Internal until the table's sizing
135    /// is settled (v0.5).
136    #[doc(hidden)]
137    pub region_capacity: u32,
138    /// The container format a create writes: 2 (the default). Format 1 is
139    /// for tests and migration fixtures only: a writable open of a format-1
140    /// store is refused (`Unsupported(FormatVersion)`; copy it to format 2
141    /// with `copy_store`), a read-only open proceeds.
142    #[doc(hidden)]
143    pub container_format: u16,
144    /// The options the `provision_{append,page}_region_default` calls (of
145    /// [`Store`] and [`Reservation`](crate::Reservation)) use (default
146    /// none: those calls are refused). An open-time choice, never stored:
147    /// each open may give others, and they apply to regions provisioned
148    /// from then on (an existing region keeps its size and cap). Checked
149    /// when used, so options a later call would refuse never fail an open.
150    pub region_defaults: Option<crate::RegionOptions>,
151}
152
153impl Default for StoreOptions {
154    fn default() -> Self {
155        Self {
156            trust: Trust::default(),
157            block_size: None,
158            log2_table_slot: 14,
159            max_io: 1 << 20,
160            buffers_per_class: 64,
161            max_write: 4 << 20,
162            large_buffers: 8,
163            queues: 0,
164            queue_depth: 64,
165            t3_pool_threads: 0,
166            append_window: 16 << 20,
167            keyed_fill: false,
168            linux_tier: LinuxTier::Auto,
169            register_file: true,
170            flushes_in_flight: 1,
171            region_capacity: 65_536,
172            container_format: store_io_format::slot::FORMAT_2,
173            region_defaults: None,
174        }
175    }
176}
177
178/// How durable writes are made durable on this store.
179#[derive(Debug, Clone, Copy)]
180pub(crate) struct Policy {
181    pub(crate) class: DurabilityClass,
182    pub(crate) label: ReceiptLabel,
183    /// Write with `dsync` (durable at completion): power-safe stores.
184    pub(crate) dsync: bool,
185    /// Barriers issue a device flush: every class except power-safe (where
186    /// only regions forcing data sync flush, D-102).
187    pub(crate) flush: bool,
188}
189
190pub use crate::report::{Repair, Unavailable, UnavailableRegion};
191
192/// What the store learned about its device, and what it decided.
193#[derive(Debug, Clone)]
194#[non_exhaustive]
195pub struct StoreReport {
196    /// Raw evidence from the platform.
197    pub evidence: Evidence,
198    /// The class decision.
199    pub decision: ClassDecision,
200    /// Block (slot) size in bytes.
201    pub block_size: u64,
202    /// Region-table capacity.
203    pub table_capacity: usize,
204    /// The volume identity.
205    pub volume: VolumeId,
206    /// Owner generation of this open (0 for read-only).
207    pub owner_generation: u64,
208    /// Whether the store was opened read-only.
209    pub read_only: bool,
210    /// The kernel interface the store's I/O uses, and why, as of create or
211    /// open. Queues made later (a `Submitter`'s, the parallel lanes, created
212    /// on first use) are counted live in [`Store::io_path`].
213    pub io: IoPath,
214    /// Regions the region table names that could not be opened, with the
215    /// reason (D-090), as the open found them (repairs since:
216    /// `Store::unavailable_regions`). Empty for a healthy store.
217    pub unavailable: Vec<UnavailableRegion>,
218    /// The container format (1 or 2).
219    pub format: u16,
220    /// Format 2: what the open found in the metadata (empty for format 1).
221    pub metadata: MetadataReport,
222    /// Flushes run at once ([`StoreOptions::flushes_in_flight`] as far as
223    /// the store and platform allow): 1 for a read-only store, a device that
224    /// needs no flush, or a platform that cannot keep flush failures per
225    /// handle; fewer where a handle for a further slot could not be made.
226    pub flushes_in_flight: u8,
227}
228
229/// In-memory metadata, guarded by the metadata lock.
230pub(crate) struct Meta {
231    pub(crate) volume: VolumeMeta,
232    pub(crate) volume_pair: PairRead,
233    pub(crate) table_pair: PairRead,
234    pub(crate) next_region_id: u32,
235    pub(crate) entries: Vec<TableEntry>,
236    pub(crate) container_len: u64,
237    /// Format 2's metadata (the format-1 fields above are then unused).
238    pub(crate) v2: Option<Box<crate::v2::Meta2>>,
239    /// The regions the table names that cannot be used, as they stand now
240    /// (the open's list, less those repaired since): lookups, the names
241    /// taken and growth read this, never the open-time report.
242    pub(crate) unavailable: Vec<UnavailableRegion>,
243    /// What the store's repairs did, in order ([`Store::repairs`]).
244    pub(crate) repairs: Vec<crate::report::Repair>,
245}
246
247/// One live region.
248pub(crate) struct LiveRegion {
249    pub(crate) id: RegionId,
250    pub(crate) name: RegionName,
251    pub(crate) kind: RegionKind,
252    pub(crate) generation: Generation,
253    /// Where the data lies in the container (every file offset of the
254    /// region's data comes from it).
255    /// Its length is the region's data size.
256    pub(crate) map: crate::regionmap::RegionMap,
257    /// The header pair's slot offsets.
258    pub(crate) header_at: [u64; 2],
259    pub(crate) header: Mutex<PairRead>,
260    pub(crate) frontier: Option<AppendFrontier>,
261    /// Append regions: whether the append position is known (true when
262    /// provisioned in this process; after a reopen, only after `resume_at`).
263    pub(crate) positioned: std::sync::atomic::AtomicBool,
264    /// Admission of data operations; closed by recycle and release.
265    pub(crate) gate: Gate,
266    /// Every barrier on the region issues a data flush, even on a
267    /// power-safe store (D-102). Runtime only: never on disk. Shared by
268    /// every generation of the region, so a setting made through any handle
269    /// holds after a recycle.
270    pub(crate) force_sync: Arc<std::sync::atomic::AtomicBool>,
271    /// Writes refused, the store being writable: format 2's regions on a
272    /// page that is not confirmed, possibly superseded, or with a header
273    /// feature this version may not write (always false in format 1).
274    /// Lifted by `accept_page_rollback` once every page is confirmed, never
275    /// for a superseded or foreign one.
276    pub(crate) read_only: std::sync::atomic::AtomicBool,
277    /// Possibly superseded (spec §9 step 4): another region's newer
278    /// footprint may cover its bytes; it stays read-only for good.
279    pub(crate) superseded: bool,
280    /// Its header's winner carries a read-only-compatible feature this
281    /// version does not know (spec §14): read-only for good too.
282    pub(crate) foreign: bool,
283}
284
285impl LiveRegion {
286    /// Whether barriers on this region always flush (D-102).
287    pub(crate) fn forced(&self) -> bool {
288        self.force_sync.load(std::sync::atomic::Ordering::Acquire)
289    }
290
291    /// Whether writes to the region are refused (see `read_only`).
292    pub(crate) fn is_read_only(&self) -> bool {
293        self.read_only.load(std::sync::atomic::Ordering::Acquire)
294    }
295}
296
297pub(crate) struct Inner<P: Platform> {
298    /// Declared first, so the queues drop first (fields drop in declaration
299    /// order): a queue whose operations run on worker threads waits for them
300    /// in its drop, and none may land after the ownership lock is released.
301    pub(crate) queues: QueueSet<P::Queue>,
302    /// The flush thread (D-099), started by the first submitter request
303    /// that needs a flush (a flush-required store, or a region forcing data
304    /// sync); `None` inside if the OS refused it. Declared
305    /// early too: it is stopped and joined before the file and the ownership
306    /// lock are released.
307    pub(crate) flusher: std::sync::OnceLock<Option<crate::flusher::Flusher<P>>>,
308    pub(crate) platform: P,
309    /// The store directory, held open for directory syncs.
310    pub(crate) _dir: P::Dir,
311    pub(crate) file: P::File,
312    /// Flush slots 1.. (D-124): a handle each, opened on the container
313    /// beside `file` (slot 0's). Empty with one slot.
314    pub(crate) flush_files: Box<[P::File]>,
315    pub(crate) _lock: P::Lock,
316    pub(crate) volume: VolumeId,
317    pub(crate) layout: Layout,
318    pub(crate) policy: Policy,
319    pub(crate) domain: Domain,
320    pub(crate) pool: BufPool,
321    pub(crate) meta: Mutex<Meta>,
322    pub(crate) regions: Mutex<Vec<Arc<LiveRegion>>>,
323    /// What the open found (never changed after: see `Meta::unavailable`
324    /// and `frozen` for what stands now).
325    pub(crate) report: StoreReport,
326    /// Allocation is frozen (format 2: a page is Unconfirmed or Lost), as
327    /// it stands now: set at open, only ever cleared afterwards (once no
328    /// page is stale), so a check made without the metadata lock can only
329    /// refuse what it might have allowed, never the reverse.
330    pub(crate) frozen: std::sync::atomic::AtomicBool,
331    pub(crate) read_only: bool,
332    pub(crate) owner_generation: u64,
333    pub(crate) opts: StoreOptions,
334    /// The largest ordinary pool buffer (`Store::ordinary`), fixed at open.
335    pub(crate) ordinary: usize,
336    /// One write's largest piece (`Store::chunk`): the ordinary buffer size,
337    /// in whole blocks; fixed at open.
338    pub(crate) chunk: usize,
339    /// Space accounting (lock order: `meta`, then `acct`).
340    pub(crate) acct: Mutex<Accounting>,
341    /// Unique per open store in this process: tickets and receipts of one
342    /// open never vouch for another's writes.
343    pub(crate) epoch: u64,
344    /// Per-stage timing and counters (zero-sized without feature `observe`).
345    pub(crate) observer: Observer,
346    /// What `oldest_outstanding` queries have seen (D-095).
347    pub(crate) seen: Mutex<crate::stats::Seen>,
348    /// Live submitters' busy marks (D-098).
349    pub(crate) submitters: Mutex<crate::stats::Submitters>,
350}
351
352/// A store: one container file on one device.
353pub struct Store<P: Platform> {
354    pub(crate) inner: Arc<Inner<P>>,
355}
356
357impl<P: Platform> core::fmt::Debug for Store<P> {
358    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
359        f.debug_struct("Store")
360            .field("volume", &self.inner.volume)
361            .field("class", &self.inner.policy.class)
362            .field("read_only", &self.inner.read_only)
363            .field("poisoned", &self.inner.domain.poisoned().is_some())
364            .finish()
365    }
366}
367
368impl<P: Platform> Clone for Store<P> {
369    fn clone(&self) -> Self {
370        Self {
371            inner: Arc::clone(&self.inner),
372        }
373    }
374}
375
376pub(crate) fn io(op: Op, raw: OsError) -> Error {
377    Error::Io {
378        op,
379        raw,
380        ctx: ErrorContext::default(),
381    }
382}
383
384pub(crate) fn lock_meta(m: &Mutex<Meta>) -> MutexGuard<'_, Meta> {
385    m.lock().unwrap_or_else(PoisonError::into_inner)
386}
387
388/// A random 128-bit identity. Uniqueness matters, secrecy does not: the
389/// standard library's OS-seeded hasher keys, mixed with the clock and the
390/// process id through SplitMix64.
391pub(crate) fn random_uuid() -> [u8; 16] {
392    use std::hash::{BuildHasher, Hasher};
393    let mut h = std::collections::hash_map::RandomState::new().build_hasher();
394    h.write_u128(
395        std::time::SystemTime::now()
396            .duration_since(std::time::UNIX_EPOCH)
397            .map_or(0, |d| d.as_nanos()),
398    );
399    h.write_u32(std::process::id());
400    let mut z = h.finish();
401    let mut out = [0u8; 16];
402    for chunk in out.chunks_exact_mut(8) {
403        z = z.wrapping_add(0x9E37_79B9_7F4A_7C15);
404        let mut x = z;
405        x = (x ^ (x >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
406        x = (x ^ (x >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
407        chunk.copy_from_slice(&(x ^ (x >> 31)).to_le_bytes());
408    }
409    out
410}
411
412impl<P: Platform> Store<P>
413where
414    P::Queue: Send,
415{
416    /// The failure that poisoned the store, if one has: every write and
417    /// barrier fails with `Poisoned` from then on. A cheap check, for a
418    /// health probe or a monitor.
419    #[must_use]
420    pub fn poisoned(&self) -> Option<FirstCause> {
421        self.inner.domain.poisoned()
422    }
423
424    /// Event counters and sampled per-stage latencies since this handle was
425    /// opened. With feature `observe` off (the default) nothing is recorded
426    /// and the snapshot is all zeros.
427    #[must_use]
428    pub fn observe(&self) -> Snapshot {
429        self.inner.observer.snapshot()
430    }
431
432    /// What the store learned about its device and what it decided, as
433    /// the open found it (never changed after: what repairs change since
434    /// is in [`Self::unavailable_regions`], [`Self::frozen`] and
435    /// [`Self::repairs`]).
436    #[must_use]
437    pub fn report(&self) -> &StoreReport {
438        &self.inner.report
439    }
440
441    /// The kernel interface the store's I/O uses now, and why: as
442    /// [`StoreReport::io`], with every queue made since counted (a ring the
443    /// kernel refused for a `Submitter` or a parallel lane, and what
444    /// served instead).
445    #[must_use]
446    pub fn io_path(&self) -> IoPath {
447        self.inner.platform.io_path()
448    }
449
450    /// The volume identity.
451    #[must_use]
452    pub fn volume(&self) -> VolumeId {
453        self.inner.volume
454    }
455
456    /// Poisons the device domain (for a caller's watchdog that decided the
457    /// device is hung). Every later write and barrier fails with
458    /// [`Error::Poisoned`]; reads keep working.
459    pub fn poison(&self) {
460        self.inner.domain.poison(FirstCause {
461            op: None,
462            raw: None,
463        });
464        for r in self.regions_snapshot() {
465            if let Some(f) = &r.frontier {
466                f.wake_all();
467            }
468        }
469    }
470
471    /// Flush-sharing counters of the device domain.
472    #[must_use]
473    pub fn domain_stats(&self) -> crate::domain::DomainStats {
474        self.inner.domain.stats()
475    }
476
477    /// Worker threads in the submit pool that runs this store's page batches
478    /// and large reads concurrently, once this store's first such call has
479    /// started its parallel queues (the pool may have been started earlier by
480    /// another store on the same platform, and stays `None` if they could
481    /// not be started);
482    /// `None` before that, or on a backend that needs no pool (IOCP, the
483    /// simulator).
484    #[must_use]
485    pub fn submit_pool_threads(&self) -> Option<usize> {
486        self.inner.queues.parallel_workers()
487    }
488
489    pub(crate) fn regions_snapshot(&self) -> Vec<Arc<LiveRegion>> {
490        self.inner
491            .regions
492            .lock()
493            .unwrap_or_else(PoisonError::into_inner)
494            .clone()
495    }
496
497    /// Names and kinds of the ready regions.
498    #[must_use]
499    pub fn regions(&self) -> Vec<(String, RegionKind)> {
500        self.regions_snapshot()
501            .iter()
502            .map(|r| (r.name.as_str().to_owned(), r.kind))
503            .collect()
504    }
505
506    pub(crate) fn find(
507        &self,
508        name: &str,
509        kind: RegionKind,
510        what: Named,
511    ) -> Result<Arc<LiveRegion>, Error> {
512        let name = RegionName::new(name).map_err(|_| Error::NotWritten {
513            cause: NotWrittenCause::InvalidName,
514            ctx: ErrorContext::default(),
515        })?;
516        if let Some(r) = self
517            .regions_snapshot()
518            .into_iter()
519            .find(|r| r.name == name && r.kind == kind)
520        {
521            return Ok(r);
522        }
523        // A region the table names but open could not use is corrupt, not
524        // missing: a caller must never mistake it for a fresh start (D-090).
525        match lock_meta(&self.inner.meta)
526            .unavailable
527            .iter()
528            .find(|u| u.name == name.as_str() && u.kind == kind)
529            .cloned()
530        {
531            Some(u) => Err(Error::Corruption {
532                kind: u.reason.corruption_kind(),
533                ctx: ErrorContext {
534                    region: Some(u.id),
535                    range: None,
536                },
537            }),
538            None => Err(Error::NotFound { what }),
539        }
540    }
541
542    /// Regions the region table names that open could not use, with the
543    /// reason (D-090), less any repaired since. Empty for a healthy store.
544    /// Their names stay reserved.
545    #[must_use]
546    pub fn unavailable_regions(&self) -> Vec<UnavailableRegion> {
547        lock_meta(&self.inner.meta).unavailable.clone()
548    }
549
550    /// Whether allocation is frozen now (format 2: a table page is stale
551    /// or lost; spec §9 step 2): no provisioning, no growth beyond
552    /// capacity, no release, no repair. `report().metadata.frozen` is what
553    /// the open found.
554    #[must_use]
555    pub fn frozen(&self) -> bool {
556        self.inner.frozen.load(std::sync::atomic::Ordering::Acquire)
557    }
558
559    /// What this handle's repairs and accepts did, in order
560    /// ([`Self::repair_unavailable`], [`Self::accept_page_rollback`]): the
561    /// store's log of them, for the caller to record. The library prints
562    /// nothing.
563    #[must_use]
564    pub fn repairs(&self) -> Vec<crate::report::Repair> {
565        lock_meta(&self.inner.meta).repairs.clone()
566    }
567}
568
569/// Writes one slot durably according to the store's policy: a `dsync` write
570/// on power-safe devices; a plain write plus a domain barrier otherwise.
571/// Returns the pair's new state, derived from `prev` and the bytes written;
572/// the device is never read back (D-069), so once the write is durable no
573/// later step can fail or report the old version as current.
574// The pair (location, previous state), the slot to write and the store's I/O
575// context are all needed and none derives from another.
576#[allow(clippy::too_many_arguments)]
577pub(crate) fn durable_slot<P: Platform>(
578    q: &mut Lane<P::Queue>,
579    file: &P::File,
580    pool: &BufPool,
581    at: &PairAt,
582    prev: &PairRead,
583    next: &NextSlot<'_>,
584    policy: &Policy,
585    domain: &Domain,
586) -> Result<PairRead, Error> {
587    if let Some(first) = domain.poisoned() {
588        return Err(Error::Poisoned { first });
589    }
590    let written = write_slot(q, file, pool, at, next, policy.dsync).inspect_err(|e| {
591        if matches!(e, Error::DurabilityUnknown { .. }) {
592            domain.poison(FirstCause {
593                op: Some(Op::Write),
594                raw: None,
595            });
596        }
597    })?;
598    if policy.flush {
599        let need = domain.ticket();
600        // Through this lane and the main handle: slot 0's.
601        domain.barrier_main(need, || flush_data(q, file))?;
602    }
603    Ok(prev.after(written))
604}
605
606/// One device flush through a queue. A flush refused without an OS error
607/// (a full queue) carries no code: none is invented.
608pub(crate) fn flush_data<Q: Queue>(q: &mut Lane<Q>, file: &Q::File) -> Result<(), Option<OsError>> {
609    match run(q, IoOp::FlushData { file }) {
610        Ok(Ran::Done { result, .. }) => result.map(|_| ()).map_err(Some),
611        Ok(Ran::Abandoned(raw)) => Err(Some(raw)),
612        Err((_b, raw)) => Err(raw),
613    }
614}
615
616pub(crate) fn container_name() -> Result<FileName, Error> {
617    FileName::new(CONTAINER).map_err(|_| Error::NotWritten {
618        cause: NotWrittenCause::InvalidName,
619        ctx: ErrorContext::default(),
620    })
621}
622
623pub(crate) fn refuse(d: &ClassDecision) -> Result<(), Error> {
624    match d.durable_open {
625        DurableOpen::Allowed | DurableOpen::Overridden => Ok(()),
626        DurableOpen::RefusedUnsafe => Err(Error::UnsafeDevice { reasons: d.reasons }),
627        DurableOpen::RefusedUnverified => Err(Error::Unverified {
628            missing: d.missing,
629            reasons: d.reasons,
630        }),
631    }
632}
633
634pub(crate) fn policy_of(d: &ClassDecision) -> Policy {
635    let power_safe = d.class == DurabilityClass::PowerSafe;
636    Policy {
637        class: d.class,
638        label: d.label,
639        dsync: power_safe,
640        flush: !power_safe,
641    }
642}
643
644pub(crate) fn make_pool(
645    ev: &Evidence,
646    layout: &Layout,
647    opts: &StoreOptions,
648    large: bool,
649) -> Result<BufPool, Error> {
650    let align = [
651        4096usize,
652        ev.fs.dio_mem_align as usize,
653        ev.device.logical_block as usize,
654    ]
655    .into_iter()
656    .max()
657    .unwrap_or(4096)
658    .next_power_of_two();
659    let min = (layout.block() as usize).max(align);
660    let max = io_max(opts).max(min);
661    let mut cfg = PoolConfig::uniform(align, min, max, opts.buffers_per_class.max(4));
662    // One class of larger buffers for single large submitter operations
663    // (D-101), fewer of them; a request between the two sizes takes one.
664    // Never in the short-lived pools that read the metadata at open.
665    let size = large_size(opts);
666    if large && opts.large_buffers > 0 && size > max {
667        cfg.classes.push(store_io_buf::ClassSpec {
668            size,
669            count: opts.large_buffers,
670        });
671    }
672    BufPool::new(&cfg).map_err(|_| Error::NotWritten {
673        cause: NotWrittenCause::PoolExhausted,
674        ctx: ErrorContext::default(),
675    })
676}
677
678/// The largest single I/O anything may ask for: 1 GiB (above 2 GiB a single
679/// `pwrite` transfers short, above 4 GiB `WriteFile` refuses).
680const LARGEST_IO: usize = 1 << 30;
681
682/// The largest ordinary pool buffer, which the blocking layers never exceed:
683/// `max_io` rounded up to a power of two, between 128 KiB and 1 GiB.
684pub(crate) fn io_max(opts: &StoreOptions) -> usize {
685    opts.max_io.clamp(2 << 16, LARGEST_IO).next_power_of_two()
686}
687
688/// The large class's buffer size: `max_write` rounded up to a power of two,
689/// at most 1 GiB (no large class when that is not above [`io_max`]).
690pub(crate) fn large_size(opts: &StoreOptions) -> usize {
691    opts.max_write.clamp(1, LARGEST_IO).next_power_of_two()
692}
693
694pub(crate) fn make_queues<P: Platform>(
695    platform: &P,
696    opts: &StoreOptions,
697    flush_lanes: usize,
698) -> Result<QueueSet<P::Queue>, Error> {
699    let n = if opts.queues == 0 {
700        std::thread::available_parallelism().map_or(4, std::num::NonZeroUsize::get)
701    } else {
702        opts.queues
703    };
704    let cfg = QueueConfig {
705        depth: opts.queue_depth.max(1),
706        ..QueueConfig::default()
707    };
708    let mut qs = Vec::with_capacity(n);
709    for _ in 0..n {
710        qs.push(platform.queue(cfg).map_err(|e| io(Op::Queue, e))?);
711    }
712    // One more queue per flush slot, used only by that slot's flush leader
713    // (D-070, D-124).
714    let mut flush = Vec::with_capacity(flush_lanes);
715    for _ in 0..flush_lanes.max(1) {
716        flush.push(platform.queue(cfg).map_err(|e| io(Op::Queue, e))?);
717    }
718    Ok(QueueSet::with_flush_lanes(qs, flush, cfg.depth as usize))
719}
720
721/// The flush slots a store runs (D-124): `opts.flushes_in_flight` clamped to
722/// `1..=MAX_FLUSH_SLOTS`, and 1 unless the platform reports flush failures
723/// per handle.
724pub(crate) fn flush_slots<P: Platform>(platform: &P, opts: &StoreOptions) -> usize {
725    let asked = usize::from(opts.flushes_in_flight).clamp(1, crate::domain::MAX_FLUSH_SLOTS);
726    if asked > 1 && platform.flush_errors_per_handle() {
727        asked
728    } else {
729        1
730    }
731}
732
733pub(crate) fn space_err(op: Op, raw: OsError, tag: u32) -> Error {
734    match classify(Stage::Space, raw) {
735        store_io_core::errno::ErrorKind::NoSpace => Error::NoSpace {
736            tag: ReserveTag::Caller(tag),
737            ctx: ErrorContext::default(),
738        },
739        _ => io(op, raw),
740    }
741}
742
743pub(crate) fn meta_err() -> Error {
744    Error::Corruption {
745        kind: store_io_core::error::CorruptionKind::Metadata,
746        ctx: ErrorContext::default(),
747    }
748}
749
750pub(crate) fn corrupt() -> Error {
751    meta_err()
752}
753
754pub(crate) fn pair_volume(volume: VolumeId, l: &Layout) -> PairAt {
755    let (a, b) = l.volume_slots();
756    PairAt {
757        volume: *volume.as_bytes(),
758        object: VOLUME_OBJECT,
759        a,
760        b,
761        log2: l.log2_block,
762        format: l.format,
763        numbered: l.format == store_io_format::slot::FORMAT_2,
764    }
765}
766
767pub(crate) fn pair_table(volume: VolumeId, l: &Layout) -> PairAt {
768    let (a, b) = l.table_slots();
769    PairAt {
770        volume: *volume.as_bytes(),
771        object: TABLE_OBJECT,
772        a,
773        b,
774        log2: l.log2_table,
775        format: l.format,
776        numbered: false,
777    }
778}
779
780pub(crate) fn pair_region(volume: VolumeId, l: &Layout, e: &TableEntry) -> PairAt {
781    PairAt {
782        volume: *volume.as_bytes(),
783        object: l.region_object(u64::from(e.region_id)),
784        a: e.offset,
785        b: e.offset + l.block(),
786        log2: l.log2_block,
787        format: l.format,
788        numbered: false,
789    }
790}