Skip to main content

store_io_engine/
store.rs

1//! [`Store`]: one container on one device.
2//!
3//! A store owns the container file, the ownership lock, the device's flush
4//! domain, the buffer pool and the queues, and the in-memory copy of the
5//! volume record and region table. Region handles share it through an `Arc`.
6//!
7//! The metadata lock (`meta`) is taken only by lifecycle operations
8//! (provision, release, lookup by name); reads and writes of region data never
9//! take it.
10
11use std::sync::{Arc, Mutex, MutexGuard, PoisonError};
12
13pub use crate::v2::{DamagedSlot, Healed, MetadataReport, SlotDamage};
14
15use store_io_buf::{BufPool, PoolConfig};
16use store_io_core::class::{DurabilityClass, ReceiptLabel};
17use store_io_core::decide::{ClassDecision, DurableOpen, Trust};
18use store_io_core::errno::{Stage, classify};
19use store_io_core::error::{
20    Error, ErrorContext, FirstCause, Named, NotWrittenCause, Op, OsError, ReserveTag,
21};
22use store_io_core::evidence::Evidence;
23use store_io_core::id::{Generation, RegionId, VolumeId};
24use store_io_core::observe::{Observer, Snapshot};
25use store_io_format::meta::{RegionKind, RegionName, TableEntry, VolumeMeta};
26use store_io_platform::{FileName, IoOp, IoPath, LinuxTier, Platform, Queue, QueueConfig};
27
28use crate::domain::Domain;
29use crate::exec::{Lane, QueueSet, Ran, run};
30use crate::frontier::AppendFrontier;
31use crate::gate::Gate;
32use crate::layout::{Layout, TABLE_OBJECT, VOLUME_OBJECT};
33use crate::slots::{NextSlot, PairAt, PairRead, write_slot};
34use crate::space::Accounting;
35
36/// The container file name inside the store directory.
37pub const CONTAINER: &str = "store.sio";
38
39/// Explicit, reported choices for a store. Every durability-affecting field
40/// appears in [`Store::report`].
41#[derive(Debug, Clone)]
42pub struct StoreOptions {
43    /// Certificate, attestation and override inputs to the class decision.
44    pub trust: Trust,
45    /// Minimum block (slot) size in bytes; the device may require more.
46    pub block_size: Option<u32>,
47    /// Region-table slot size as a power of two (14 = 16 KiB: a format-2
48    /// table page of 169 regions; format 1's single table, 253).
49    pub log2_table_slot: u8,
50    /// Largest single I/O of the blocking layers, and the batch buffer size
51    /// (bytes; rounded up to a power of two, 128 KiB to 1 GiB).
52    pub max_io: usize,
53    /// Buffers per size class in the pool, up to `max_io` (at most
54    /// 16,777,215; more fails `create` and `open` with
55    /// `NotWritten(PoolExhausted)`).
56    pub buffers_per_class: u32,
57    /// Largest single write or read a `Submitter` takes, from one buffer, as
58    /// one OS write or read operation (bytes; rounded up to a power of two,
59    /// at most 1 GiB): the pool holds `large_buffers` buffers of that size,
60    /// used by submitters only (D-101). Ignored when not above `max_io`.
61    pub max_write: usize,
62    /// Buffers of `max_write` bytes in the pool (0: none, so a submitter's
63    /// largest operation is `max_io`; at most 16,777,215). The defaults add
64    /// 32 MiB on top of about 128 MiB of ordinary buffers per store: Windows
65    /// charges the whole pool to the commit limit at creation; on both
66    /// systems a buffer becomes resident when first used, and the most
67    /// recently returned buffer of a size is reused first, so only about as
68    /// many as are in use at once become resident. A pool too large to map
69    /// fails `create` and `open` with `NotWritten(PoolExhausted)`.
70    pub large_buffers: u32,
71    /// Queues shared by the blocking layers (0 = number of CPUs). On Linux
72    /// they perform their operations inline on either tier; the parallel
73    /// lanes, as many again and created on first use, are io_uring rings on
74    /// the io_uring tier, each holding two file descriptors (its ring and
75    /// its eventfd), as does each `Submitter`.
76    pub queues: usize,
77    /// Operations in flight per queue.
78    pub queue_depth: u32,
79    /// Worker threads for the platform's submit pool, which runs the
80    /// independent operations of a page batch or a large read concurrently
81    /// on the Linux synchronous tier (D-051, D-096); 0 = one per CPU, at most
82    /// 32. The pool is shared by every store on the platform and sized by the
83    /// first store that uses it (at most 256); other backends ignore this.
84    pub t3_pool_threads: u32,
85    /// How far, in bytes, a new append may start ahead of the oldest
86    /// incomplete one on the same region, in blocks: rounded down to a power
87    /// of two, or, on a region that cannot grow past a size smaller than the
88    /// window (format 1's regions, a capped format-2 region), that size
89    /// rounded up (so a region never limits its own appends); at least 2 and
90    /// at most 2^20 blocks. Each append region that has appended since the
91    /// open or its last recycle holds 8 bytes of memory per block of its
92    /// ring: a format-2 region that may grow (no `max_size`) holds the
93    /// window's whatever its size, 32 KiB at the default 16 MiB and 4 KiB
94    /// blocks (2 GiB for 65,536 such regions; a cap, or a smaller window,
95    /// bounds it). Default 16 MiB.
96    pub append_window: u64,
97    /// Fill new regions with the keyed incompressible pattern instead of
98    /// zeros (for thin, virtual, deduplicating or zero-detecting stacks).
99    pub keyed_fill: bool,
100    /// The kernel interface on Linux (D-082): `Auto` (io_uring where the
101    /// kernel accepts it), `Sync` or `Uring`. Read by the native API when
102    /// it makes the platform; the choice is in [`StoreReport::io`]. Other
103    /// systems ignore it.
104    pub linux_tier: LinuxTier,
105    /// Register the store's container file with each `Submitter`'s queue
106    /// where the backend can (D-078, D-107: on Linux's io_uring tier, a
107    /// read-only twin of it as a direct descriptor on the submitter's ring),
108    /// so its reads skip the per-operation file lookup in the kernel; writes
109    /// stay on the store's own descriptor, so a write in flight keeps the
110    /// store locked until it completes. Best effort: a
111    /// registration the kernel refuses is counted in `Store::io_path`, never
112    /// an error. On by default: the gates measured reads 1.03x to 1.08x of
113    /// an unregistered ring (G3d, CI); writes take the unregistered path.
114    /// Buffers
115    /// are never registered: reads into registered buffers hang forever on
116    /// some kernels (at least Linux 6.5.0 and 6.6.0), which no probe
117    /// detects (D-107).
118    pub register_file: bool,
119    /// Flushes the store runs at once (1 to 4; default 1, one in flight as
120    /// before, D-124). Several can overlap, meant for drives that execute
121    /// flushes in parallel (NVMe; not yet measured on one): each runs on a
122    /// handle and queue of its own, so each reports its own failures, and a
123    /// receipt still needs every flush up to its own to have succeeded.
124    /// Platforms that cannot keep flush failures per handle run one (Linux
125    /// 4.16 and later can); the number run is in
126    /// [`StoreReport::flushes_in_flight`]. Measure on the target drive
127    /// before raising it: at eight writers, 2 and 4 gained nothing on a
128    /// memory-backed device and lost on a virtual disk (1,314, 974 and 595
129    /// durable writes a second at 1, 2 and 4).
130    pub flushes_in_flight: u8,
131    /// Regions a format-2 store can hold, fixed at create (default 65,536:
132    /// a table of 12.1 MiB at 16 KiB table pages, written at create and
133    /// read at every open; smaller stores may ask for fewer; at most
134    /// 4,194,304). Ignored by format 1. Internal until the table's sizing
135    /// is settled (v0.5).
136    #[doc(hidden)]
137    pub region_capacity: u32,
138    /// The container format a create writes: 2 (the default). Format 1 is
139    /// for tests and migration fixtures only: a writable open of a format-1
140    /// store is refused (`Unsupported(FormatVersion)`; copy it to format 2
141    /// with `copy_store`), a read-only open proceeds.
142    #[doc(hidden)]
143    pub container_format: u16,
144    /// The options the `provision_{append,page}_region_default` calls (of
145    /// [`Store`] and [`Reservation`](crate::Reservation)) use (default
146    /// none: those calls are refused). An open-time choice, never stored:
147    /// each open may give others, and they apply to regions provisioned
148    /// from then on (an existing region keeps its size and cap). Checked
149    /// when used, so options a later call would refuse never fail an open.
150    pub region_defaults: Option<crate::RegionOptions>,
151    /// Device pacing (PLANNING R5.4): background work (scans, copies,
152    /// background `Submitter`s; paced as each lands in v0.6) is split into
153    /// pieces of this many bytes, whatever the device's state, except a
154    /// copy's fill of its target, which keeps one piece in flight: while
155    /// commits are quiet its pieces are `background_unpaced` of these, at
156    /// most 1 MiB (as many bytes as a deeper call may then keep in flight;
157    /// D-156). A multiple of
158    /// 4096 from 4096 to 256 KiB (PERF-10's piece); default 256 KiB. The
159    /// store splits at the smaller of this and its largest I/O (`max_io`
160    /// rounded up to a power of two, at least 128 KiB), rounded down to
161    /// whole blocks, at least one; [`crate::QosReport::background_io`] says
162    /// what it uses.
163    pub background_io: u32,
164    /// Background pieces in flight per background call while the device's
165    /// commits are not active: 1 to 64; default 8. Stores of one device take
166    /// the smallest.
167    pub background_unpaced: u32,
168    /// Pace background work while the device's commits are active (default
169    /// `true`): one piece in flight per background call. Off for
170    /// measurements such as G19's negative control; stores of one device
171    /// pace together if any of them asks for it.
172    pub background_pacing: bool,
173    /// How long after a commit the device's commits count as active: at
174    /// least 1 ms; default 5 s. Stores of one device take the largest.
175    pub commit_quiet: std::time::Duration,
176    /// Opt-in auto-tuning of the device's background budget (PLANNING
177    /// R5.4; default `false`). While commits are active, the pieces the
178    /// device group admits beyond each background call's own one follow
179    /// windows of commit latency (200 commits, or 2 s with at least 50),
180    /// each compared with a window run at the floor (1) in the same
181    /// conditions, not with an idle figure: the budget rises by one after
182    /// three windows within 1.25x of the latest floor window's p99, halves
183    /// at once on one over 1.4x, and every eighth window measures the floor
184    /// again, up to `background_depth_max`. Its bound, documented and not
185    /// gated: commits may run at up to 1.4x the floor's own latency for as
186    /// long as background work runs (the default, off, keeps the floor,
187    /// which G19 gates within 2x of idle). On only if every store of the
188    /// device asks for it. A device whose commits are only a power-safe
189    /// store's asynchronous `Submitter` writes is never sampled, so stays
190    /// at the floor. [`crate::QosReport`] says what it does.
191    pub background_autotune: bool,
192    /// Auto-tuning's ceiling: the background pieces the device group may
193    /// admit while commits are active (1 to 8; default 8), and never more
194    /// than `background_unpaced`. Stores of one device take the smallest.
195    pub background_depth_max: u32,
196}
197
198impl Default for StoreOptions {
199    fn default() -> Self {
200        Self {
201            trust: Trust::default(),
202            block_size: None,
203            log2_table_slot: 14,
204            max_io: 1 << 20,
205            buffers_per_class: 64,
206            max_write: 4 << 20,
207            large_buffers: 8,
208            queues: 0,
209            queue_depth: 64,
210            t3_pool_threads: 0,
211            append_window: 16 << 20,
212            keyed_fill: false,
213            linux_tier: LinuxTier::Auto,
214            register_file: true,
215            flushes_in_flight: 1,
216            region_capacity: 65_536,
217            container_format: store_io_format::slot::FORMAT_2,
218            region_defaults: None,
219            background_io: 256 << 10,
220            background_unpaced: 8,
221            background_autotune: false,
222            background_depth_max: 8,
223            background_pacing: true,
224            commit_quiet: std::time::Duration::from_secs(5),
225        }
226    }
227}
228
229/// Checks the pacing options (PLANNING R5.4): `NotWritten(InvalidOptions)`
230/// for a `background_io` that is not a multiple of 4096 from 4096 to
231/// 256 KiB, a `background_unpaced` outside 1 to 64, a
232/// `background_depth_max` outside 1 to 8, or a `commit_quiet` under 1 ms.
233pub(crate) fn check_pacing(opts: &StoreOptions) -> Result<(), Error> {
234    let io = opts.background_io as usize;
235    let ok = io >= 4096
236        && io % 4096 == 0
237        && io <= 256 << 10
238        && (1..=64).contains(&opts.background_unpaced)
239        && (1..=8).contains(&opts.background_depth_max)
240        && opts.commit_quiet >= std::time::Duration::from_millis(1);
241    if ok {
242        Ok(())
243    } else {
244        Err(Error::NotWritten {
245            cause: store_io_core::error::NotWrittenCause::InvalidOptions,
246            ctx: ErrorContext::default(),
247        })
248    }
249}
250
251/// The pacing options a store with blocks of `block` bytes joins its device
252/// group with: its piece never past its largest I/O, and whole blocks (at
253/// least one), so a piece is never a misaligned or partial-block I/O.
254pub(crate) fn qos_options(opts: &StoreOptions, block: u64) -> crate::qos::QosOptions {
255    let block = u32::try_from(block).unwrap_or(u32::MAX).max(1);
256    let piece = opts
257        .background_io
258        .min(u32::try_from(io_max(opts)).unwrap_or(u32::MAX));
259    crate::qos::QosOptions {
260        pacing: opts.background_pacing,
261        commit_quiet_ns: u64::try_from(opts.commit_quiet.as_nanos()).unwrap_or(u64::MAX),
262        background_io: (piece / block * block).max(block),
263        background_unpaced: opts.background_unpaced,
264        autotune: opts.background_autotune,
265        depth_max: opts.background_depth_max,
266    }
267}
268
269/// Joins the device group a store with `ev` and `opts` belongs to (R5.4):
270/// never fails; what the evidence does not name is paced the safe way.
271pub(crate) fn join_group<P: Platform>(
272    platform: &P,
273    ev: &Evidence,
274    opts: &StoreOptions,
275    block: u64,
276) -> Arc<crate::qos::Qos> {
277    crate::qos::Registry::global().join(
278        platform.identity_scope(),
279        &placement(ev),
280        qos_options(opts, block),
281        platform.monotonic_ns(),
282    )
283}
284
285/// Where a store with `ev` joins pacing: a memory file system joins no
286/// group; anything else by its device's keys.
287pub(crate) fn placement(ev: &Evidence) -> crate::qos::Placement {
288    if ev.fs.kind == store_io_core::evidence::FsKind::Tmpfs {
289        return crate::qos::Placement::Memory;
290    }
291    crate::qos::Placement::Keyed {
292        identity: ev.device.identity.clone(),
293        members: ev.device.members.clone(),
294        fs_device: ev.fs.device,
295    }
296}
297
298/// How durable writes are made durable on this store.
299#[derive(Debug, Clone, Copy)]
300pub(crate) struct Policy {
301    pub(crate) class: DurabilityClass,
302    pub(crate) label: ReceiptLabel,
303    /// Write with `dsync` (durable at completion): power-safe stores.
304    pub(crate) dsync: bool,
305    /// Barriers issue a device flush: every class except power-safe (where
306    /// only regions forcing data sync flush, D-102).
307    pub(crate) flush: bool,
308}
309
310pub use crate::report::{Repair, Unavailable, UnavailableRegion};
311
312/// What the store learned about its device, and what it decided.
313#[derive(Debug, Clone)]
314#[non_exhaustive]
315pub struct StoreReport {
316    /// Raw evidence from the platform.
317    pub evidence: Evidence,
318    /// The class decision.
319    pub decision: ClassDecision,
320    /// Block (slot) size in bytes.
321    pub block_size: u64,
322    /// Region-table capacity.
323    pub table_capacity: usize,
324    /// The volume identity.
325    pub volume: VolumeId,
326    /// Owner generation of this open (0 for read-only).
327    pub owner_generation: u64,
328    /// Whether the store was opened read-only.
329    pub read_only: bool,
330    /// The kernel interface the store's I/O uses, and why, as of create or
331    /// open. Queues made later (a `Submitter`'s, the parallel lanes, created
332    /// on first use) are counted live in [`Store::io_path`].
333    pub io: IoPath,
334    /// Regions the region table names that could not be opened, with the
335    /// reason (D-090), as the open found them (repairs since:
336    /// `Store::unavailable_regions`). Empty for a healthy store.
337    pub unavailable: Vec<UnavailableRegion>,
338    /// The container format (1 or 2).
339    pub format: u16,
340    /// Format 2: what the open found in the metadata (empty for format 1).
341    pub metadata: MetadataReport,
342    /// Flushes run at once ([`StoreOptions::flushes_in_flight`] as far as
343    /// the store and platform allow): 1 for a read-only store, a device that
344    /// needs no flush, or a platform that cannot keep flush failures per
345    /// handle; fewer where a handle for a further slot could not be made.
346    pub flushes_in_flight: u8,
347}
348
349/// In-memory metadata, guarded by the metadata lock.
350pub(crate) struct Meta {
351    pub(crate) volume: VolumeMeta,
352    pub(crate) volume_pair: PairRead,
353    pub(crate) table_pair: PairRead,
354    pub(crate) next_region_id: u32,
355    pub(crate) entries: Vec<TableEntry>,
356    pub(crate) container_len: u64,
357    /// Format 2's metadata (the format-1 fields above are then unused).
358    pub(crate) v2: Option<Box<crate::v2::Meta2>>,
359    /// The regions the table names that cannot be used, as they stand now
360    /// (the open's list, less those repaired since): lookups, the names
361    /// taken and growth read this, never the open-time report.
362    pub(crate) unavailable: Vec<UnavailableRegion>,
363    /// What the store's repairs did, in order ([`Store::repairs`]).
364    pub(crate) repairs: Vec<crate::report::Repair>,
365}
366
367/// One live region.
368pub(crate) struct LiveRegion {
369    pub(crate) id: RegionId,
370    pub(crate) name: RegionName,
371    pub(crate) kind: RegionKind,
372    pub(crate) generation: Generation,
373    /// Where the data lies in the container (every file offset of the
374    /// region's data comes from it).
375    /// Its length is the region's data size.
376    pub(crate) map: crate::regionmap::RegionMap,
377    /// The header pair's slot offsets.
378    pub(crate) header_at: [u64; 2],
379    pub(crate) header: Mutex<PairRead>,
380    pub(crate) frontier: Option<AppendFrontier>,
381    /// Append regions: whether the append position is known (true when
382    /// provisioned in this process; after a reopen, only after `resume_at`).
383    pub(crate) positioned: std::sync::atomic::AtomicBool,
384    /// Admission of data operations; closed by recycle and release.
385    pub(crate) gate: Gate,
386    /// Every barrier on the region issues a data flush, even on a
387    /// power-safe store (D-102). Runtime only: never on disk. Shared by
388    /// every generation of the region, so a setting made through any handle
389    /// holds after a recycle.
390    pub(crate) force_sync: Arc<std::sync::atomic::AtomicBool>,
391    /// Writes refused, the store being writable: format 2's regions on a
392    /// page that is not confirmed, possibly superseded, or with a header
393    /// feature this version may not write (always false in format 1).
394    /// Lifted by `accept_page_rollback` once every page is confirmed, never
395    /// for a superseded or foreign one.
396    pub(crate) read_only: std::sync::atomic::AtomicBool,
397    /// Possibly superseded (spec §9 step 4): another region's newer
398    /// footprint may cover its bytes; it stays read-only for good.
399    pub(crate) superseded: bool,
400    /// Its header's winner carries a read-only-compatible feature this
401    /// version does not know (spec §14): read-only for good too.
402    pub(crate) foreign: bool,
403}
404
405impl LiveRegion {
406    /// Whether barriers on this region always flush (D-102).
407    pub(crate) fn forced(&self) -> bool {
408        self.force_sync.load(std::sync::atomic::Ordering::Acquire)
409    }
410
411    /// Whether writes to the region are refused (see `read_only`).
412    pub(crate) fn is_read_only(&self) -> bool {
413        self.read_only.load(std::sync::atomic::Ordering::Acquire)
414    }
415}
416
417pub(crate) struct Inner<P: Platform> {
418    /// Declared first, so the queues drop first (fields drop in declaration
419    /// order): a queue whose operations run on worker threads waits for them
420    /// in its drop, and none may land after the ownership lock is released.
421    pub(crate) queues: QueueSet<P::Queue>,
422    /// The flush thread (D-099), started by the first submitter request
423    /// that needs a flush (a flush-required store, or a region forcing data
424    /// sync); `None` inside if the OS refused it. Declared
425    /// early too: it is stopped and joined before the file and the ownership
426    /// lock are released.
427    pub(crate) flusher: std::sync::OnceLock<Option<crate::flusher::Flusher<P>>>,
428    pub(crate) platform: P,
429    /// The store directory, held open for directory syncs.
430    pub(crate) _dir: P::Dir,
431    pub(crate) file: P::File,
432    /// Flush slots 1.. (D-124): a handle each, opened on the container
433    /// beside `file` (slot 0's). Empty with one slot.
434    pub(crate) flush_files: Box<[P::File]>,
435    pub(crate) _lock: P::Lock,
436    pub(crate) volume: VolumeId,
437    pub(crate) layout: Layout,
438    pub(crate) policy: Policy,
439    pub(crate) domain: Domain,
440    /// The store's device group (PLANNING R5.4); the domain stamps its
441    /// commit activity.
442    pub(crate) qos: Arc<crate::qos::Qos>,
443    pub(crate) pool: BufPool,
444    pub(crate) meta: Mutex<Meta>,
445    pub(crate) regions: Mutex<Vec<Arc<LiveRegion>>>,
446    /// What the open found (never changed after: see `Meta::unavailable`
447    /// and `frozen` for what stands now).
448    pub(crate) report: StoreReport,
449    /// Allocation is frozen (format 2: a page is Unconfirmed or Lost), as
450    /// it stands now: set at open, only ever cleared afterwards (once no
451    /// page is stale), so a check made without the metadata lock can only
452    /// refuse what it might have allowed, never the reverse.
453    pub(crate) frozen: std::sync::atomic::AtomicBool,
454    pub(crate) read_only: bool,
455    pub(crate) owner_generation: u64,
456    pub(crate) opts: StoreOptions,
457    /// The largest ordinary pool buffer (`Store::ordinary`), fixed at open.
458    pub(crate) ordinary: usize,
459    /// One write's largest piece (`Store::chunk`): the ordinary buffer size,
460    /// in whole blocks; fixed at open.
461    pub(crate) chunk: usize,
462    /// Space accounting (lock order: `meta`, then `acct`).
463    pub(crate) acct: Mutex<Accounting>,
464    /// Unique per open store in this process: tickets and receipts of one
465    /// open never vouch for another's writes.
466    pub(crate) epoch: u64,
467    /// Per-stage timing and counters (zero-sized without feature `observe`).
468    pub(crate) observer: Observer,
469    /// What `oldest_outstanding` queries have seen (D-095).
470    pub(crate) seen: Mutex<crate::stats::Seen>,
471    /// Live submitters' busy marks (D-098).
472    pub(crate) submitters: Mutex<crate::stats::Submitters>,
473}
474
475/// A store: one container file on one device.
476pub struct Store<P: Platform> {
477    pub(crate) inner: Arc<Inner<P>>,
478}
479
480impl<P: Platform> core::fmt::Debug for Store<P> {
481    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
482        f.debug_struct("Store")
483            .field("volume", &self.inner.volume)
484            .field("class", &self.inner.policy.class)
485            .field("read_only", &self.inner.read_only)
486            .field("poisoned", &self.inner.domain.poisoned().is_some())
487            .finish()
488    }
489}
490
491impl<P: Platform> Clone for Store<P> {
492    fn clone(&self) -> Self {
493        Self {
494            inner: Arc::clone(&self.inner),
495        }
496    }
497}
498
499pub(crate) fn io(op: Op, raw: OsError) -> Error {
500    Error::Io {
501        op,
502        raw,
503        ctx: ErrorContext::default(),
504    }
505}
506
507pub(crate) fn lock_meta(m: &Mutex<Meta>) -> MutexGuard<'_, Meta> {
508    m.lock().unwrap_or_else(PoisonError::into_inner)
509}
510
511/// A random 128-bit identity. Uniqueness matters, secrecy does not: the
512/// standard library's OS-seeded hasher keys, mixed with the clock and the
513/// process id through SplitMix64.
514pub(crate) fn random_uuid() -> [u8; 16] {
515    use std::hash::{BuildHasher, Hasher};
516    let mut h = std::collections::hash_map::RandomState::new().build_hasher();
517    h.write_u128(
518        std::time::SystemTime::now()
519            .duration_since(std::time::UNIX_EPOCH)
520            .map_or(0, |d| d.as_nanos()),
521    );
522    h.write_u32(std::process::id());
523    let mut z = h.finish();
524    let mut out = [0u8; 16];
525    for chunk in out.chunks_exact_mut(8) {
526        z = z.wrapping_add(0x9E37_79B9_7F4A_7C15);
527        let mut x = z;
528        x = (x ^ (x >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
529        x = (x ^ (x >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
530        chunk.copy_from_slice(&(x ^ (x >> 31)).to_le_bytes());
531    }
532    out
533}
534
535impl<P: Platform> Store<P>
536where
537    P::Queue: Send,
538{
539    /// The failure that poisoned the store, if one has: every write and
540    /// barrier fails with `Poisoned` from then on. A cheap check, for a
541    /// health probe or a monitor.
542    #[must_use]
543    pub fn poisoned(&self) -> Option<FirstCause> {
544        self.inner.domain.poisoned()
545    }
546
547    /// Event counters and sampled per-stage latencies since this handle was
548    /// opened. With feature `observe` off (the default) nothing is recorded
549    /// and the snapshot is all zeros.
550    #[must_use]
551    pub fn observe(&self) -> Snapshot {
552        self.inner.observer.snapshot()
553    }
554
555    /// What the store learned about its device and what it decided, as
556    /// the open found it (never changed after: what repairs change since
557    /// is in [`Self::unavailable_regions`], [`Self::frozen`] and
558    /// [`Self::repairs`]).
559    #[must_use]
560    pub fn report(&self) -> &StoreReport {
561        &self.inner.report
562    }
563
564    /// Keeps this store's device group's commits active until the hold
565    /// drops (PLANNING R5.4), so background work on the device (scans,
566    /// copies, background `Submitter`s, of this store or another on the
567    /// device) stays paced at one piece in flight per call. For a layer
568    /// that commits rarely but needs every commit fast: it holds one while
569    /// its transactions are open and drops it when idle. The cost: while a
570    /// hold lives, background work runs at the paced rate.
571    pub fn commit_hold(&self) -> crate::CommitHold {
572        self.inner.qos.hold()
573    }
574
575    /// This store's device pacing now: whether pacing is on, whether the
576    /// device's commits are active, the budget, the background pieces in
577    /// flight in the device group.
578    #[must_use]
579    pub fn qos(&self) -> crate::QosReport {
580        self.inner.qos.report(self.inner.platform.monotonic_ns())
581    }
582
583    /// The kernel interface the store's I/O uses now, and why: as
584    /// [`StoreReport::io`], with every queue made since counted (a ring the
585    /// kernel refused for a `Submitter` or a parallel lane, and what
586    /// served instead).
587    #[must_use]
588    pub fn io_path(&self) -> IoPath {
589        self.inner.platform.io_path()
590    }
591
592    /// The volume identity.
593    #[must_use]
594    pub fn volume(&self) -> VolumeId {
595        self.inner.volume
596    }
597
598    /// Poisons the device domain (for a caller's watchdog that decided the
599    /// device is hung). Every later write and barrier fails with
600    /// [`Error::Poisoned`]; reads keep working.
601    pub fn poison(&self) {
602        self.inner.domain.poison(FirstCause {
603            op: None,
604            raw: None,
605        });
606        for r in self.regions_snapshot() {
607            if let Some(f) = &r.frontier {
608                f.wake_all();
609            }
610        }
611    }
612
613    /// Flush-sharing counters of the device domain.
614    #[must_use]
615    pub fn domain_stats(&self) -> crate::domain::DomainStats {
616        self.inner.domain.stats()
617    }
618
619    /// Worker threads in the submit pool that runs this store's page batches
620    /// and large reads concurrently, once this store's first such call has
621    /// started its parallel queues (the pool may have been started earlier by
622    /// another store on the same platform, and stays `None` if they could
623    /// not be started);
624    /// `None` before that, or on a backend that needs no pool (IOCP, the
625    /// simulator).
626    #[must_use]
627    pub fn submit_pool_threads(&self) -> Option<usize> {
628        self.inner.queues.parallel_workers()
629    }
630
631    pub(crate) fn regions_snapshot(&self) -> Vec<Arc<LiveRegion>> {
632        self.inner
633            .regions
634            .lock()
635            .unwrap_or_else(PoisonError::into_inner)
636            .clone()
637    }
638
639    /// Names and kinds of the ready regions.
640    #[must_use]
641    pub fn regions(&self) -> Vec<(String, RegionKind)> {
642        self.regions_snapshot()
643            .iter()
644            .map(|r| (r.name.as_str().to_owned(), r.kind))
645            .collect()
646    }
647
648    pub(crate) fn find(
649        &self,
650        name: &str,
651        kind: RegionKind,
652        what: Named,
653    ) -> Result<Arc<LiveRegion>, Error> {
654        let name = RegionName::new(name).map_err(|_| Error::NotWritten {
655            cause: NotWrittenCause::InvalidName,
656            ctx: ErrorContext::default(),
657        })?;
658        if let Some(r) = self
659            .regions_snapshot()
660            .into_iter()
661            .find(|r| r.name == name && r.kind == kind)
662        {
663            return Ok(r);
664        }
665        // A region the table names but open could not use is corrupt, not
666        // missing: a caller must never mistake it for a fresh start (D-090).
667        match lock_meta(&self.inner.meta)
668            .unavailable
669            .iter()
670            .find(|u| u.name == name.as_str() && u.kind == kind)
671            .cloned()
672        {
673            Some(u) => Err(Error::Corruption {
674                kind: u.reason.corruption_kind(),
675                ctx: ErrorContext {
676                    region: Some(u.id),
677                    range: None,
678                },
679            }),
680            None => Err(Error::NotFound { what }),
681        }
682    }
683
684    /// Regions the region table names that open could not use, with the
685    /// reason (D-090), less any repaired since. Empty for a healthy store.
686    /// Their names stay reserved.
687    #[must_use]
688    pub fn unavailable_regions(&self) -> Vec<UnavailableRegion> {
689        lock_meta(&self.inner.meta).unavailable.clone()
690    }
691
692    /// Whether allocation is frozen now (format 2: a table page is stale
693    /// or lost; spec §9 step 2): no provisioning, no growth beyond
694    /// capacity, no release, no repair. `report().metadata.frozen` is what
695    /// the open found.
696    #[must_use]
697    pub fn frozen(&self) -> bool {
698        self.inner.frozen.load(std::sync::atomic::Ordering::Acquire)
699    }
700
701    /// What this handle's repairs and accepts did, in order
702    /// ([`Self::repair_unavailable`], [`Self::accept_page_rollback`]): the
703    /// store's log of them, for the caller to record. The library prints
704    /// nothing.
705    #[must_use]
706    pub fn repairs(&self) -> Vec<crate::report::Repair> {
707        lock_meta(&self.inner.meta).repairs.clone()
708    }
709}
710
711/// Writes one slot durably according to the store's policy: a `dsync` write
712/// on power-safe devices; a plain write plus a domain barrier otherwise.
713/// Returns the pair's new state, derived from `prev` and the bytes written;
714/// the device is never read back (D-069), so once the write is durable no
715/// later step can fail or report the old version as current.
716// The pair (location, previous state), the slot to write and the store's I/O
717// context are all needed and none derives from another.
718#[allow(clippy::too_many_arguments)]
719pub(crate) fn durable_slot<P: Platform>(
720    q: &mut Lane<P::Queue>,
721    file: &P::File,
722    pool: &BufPool,
723    at: &PairAt,
724    prev: &PairRead,
725    next: &NextSlot<'_>,
726    policy: &Policy,
727    domain: &Domain,
728) -> Result<PairRead, Error> {
729    if let Some(first) = domain.poisoned() {
730        return Err(Error::Poisoned { first });
731    }
732    // On a power-safe store the `dsync` write is the slot's flush: a
733    // commit's marks the device's commits active (a background call's
734    // marks nothing: `Domain::note_commit`).
735    let entry = if policy.dsync {
736        domain.note_commit()
737    } else {
738        None
739    };
740    let written =
741        write_slot(q, file, pool, at, next, policy.dsync, domain.pace()).inspect_err(|e| {
742            if matches!(e, Error::DurabilityUnknown { .. }) {
743                domain.poison(FirstCause {
744                    op: Some(Op::Write),
745                    raw: None,
746                });
747            }
748        })?;
749    // The power-safe slot write was the commit: auto-tuning's sample.
750    domain.sample_since(entry);
751    if policy.flush {
752        let need = domain.ticket();
753        // Through this lane and the main handle: slot 0's.
754        domain.barrier_main(need, || flush_data(q, file))?;
755    }
756    Ok(prev.after(written))
757}
758
759/// One device flush through a queue. A flush refused without an OS error
760/// (a full queue) carries no code: none is invented.
761pub(crate) fn flush_data<Q: Queue>(q: &mut Lane<Q>, file: &Q::File) -> Result<(), Option<OsError>> {
762    match run(q, IoOp::FlushData { file }) {
763        Ok(Ran::Done { result, .. }) => result.map(|_| ()).map_err(Some),
764        Ok(Ran::Abandoned(raw)) => Err(Some(raw)),
765        Err((_b, raw)) => Err(raw),
766    }
767}
768
769pub(crate) fn container_name() -> Result<FileName, Error> {
770    FileName::new(CONTAINER).map_err(|_| Error::NotWritten {
771        cause: NotWrittenCause::InvalidName,
772        ctx: ErrorContext::default(),
773    })
774}
775
776pub(crate) fn refuse(d: &ClassDecision) -> Result<(), Error> {
777    match d.durable_open {
778        DurableOpen::Allowed | DurableOpen::Overridden => Ok(()),
779        DurableOpen::RefusedUnsafe => Err(Error::UnsafeDevice { reasons: d.reasons }),
780        DurableOpen::RefusedUnverified => Err(Error::Unverified {
781            missing: d.missing,
782            reasons: d.reasons,
783        }),
784    }
785}
786
787pub(crate) fn policy_of(d: &ClassDecision) -> Policy {
788    let power_safe = d.class == DurabilityClass::PowerSafe;
789    Policy {
790        class: d.class,
791        label: d.label,
792        dsync: power_safe,
793        flush: !power_safe,
794    }
795}
796
797pub(crate) fn make_pool(
798    ev: &Evidence,
799    layout: &Layout,
800    opts: &StoreOptions,
801    large: bool,
802) -> Result<BufPool, Error> {
803    let align = [
804        4096usize,
805        ev.fs.dio_mem_align as usize,
806        ev.device.logical_block as usize,
807    ]
808    .into_iter()
809    .max()
810    .unwrap_or(4096)
811    .next_power_of_two();
812    let min = (layout.block() as usize).max(align);
813    let max = io_max(opts).max(min);
814    let mut cfg = PoolConfig::uniform(align, min, max, opts.buffers_per_class.max(4));
815    // One class of larger buffers for single large submitter operations
816    // (D-101), fewer of them; a request between the two sizes takes one.
817    // Never in the short-lived pools that read the metadata at open.
818    let size = large_size(opts);
819    if large && opts.large_buffers > 0 && size > max {
820        cfg.classes.push(store_io_buf::ClassSpec {
821            size,
822            count: opts.large_buffers,
823        });
824    }
825    BufPool::new(&cfg).map_err(|_| Error::NotWritten {
826        cause: NotWrittenCause::PoolExhausted,
827        ctx: ErrorContext::default(),
828    })
829}
830
831/// The largest single I/O anything may ask for: 1 GiB (above 2 GiB a single
832/// `pwrite` transfers short, above 4 GiB `WriteFile` refuses).
833const LARGEST_IO: usize = 1 << 30;
834
835/// The largest ordinary pool buffer, which the blocking layers never exceed:
836/// `max_io` rounded up to a power of two, between 128 KiB and 1 GiB.
837pub(crate) fn io_max(opts: &StoreOptions) -> usize {
838    opts.max_io.clamp(2 << 16, LARGEST_IO).next_power_of_two()
839}
840
841/// The large class's buffer size: `max_write` rounded up to a power of two,
842/// at most 1 GiB (no large class when that is not above [`io_max`]).
843pub(crate) fn large_size(opts: &StoreOptions) -> usize {
844    opts.max_write.clamp(1, LARGEST_IO).next_power_of_two()
845}
846
847pub(crate) fn make_queues<P: Platform>(
848    platform: &P,
849    opts: &StoreOptions,
850    flush_lanes: usize,
851) -> Result<QueueSet<P::Queue>, Error> {
852    let n = if opts.queues == 0 {
853        std::thread::available_parallelism().map_or(4, std::num::NonZeroUsize::get)
854    } else {
855        opts.queues
856    };
857    let cfg = QueueConfig {
858        depth: opts.queue_depth.max(1),
859        ..QueueConfig::default()
860    };
861    let mut qs = Vec::with_capacity(n);
862    for _ in 0..n {
863        qs.push(platform.queue(cfg).map_err(|e| io(Op::Queue, e))?);
864    }
865    // One more queue per flush slot, used only by that slot's flush leader
866    // (D-070, D-124).
867    let mut flush = Vec::with_capacity(flush_lanes);
868    for _ in 0..flush_lanes.max(1) {
869        flush.push(platform.queue(cfg).map_err(|e| io(Op::Queue, e))?);
870    }
871    Ok(QueueSet::with_flush_lanes(qs, flush, cfg.depth as usize))
872}
873
874/// The flush slots a store runs (D-124): `opts.flushes_in_flight` clamped to
875/// `1..=MAX_FLUSH_SLOTS`, and 1 unless the platform reports flush failures
876/// per handle.
877pub(crate) fn flush_slots<P: Platform>(platform: &P, opts: &StoreOptions) -> usize {
878    let asked = usize::from(opts.flushes_in_flight).clamp(1, crate::domain::MAX_FLUSH_SLOTS);
879    if asked > 1 && platform.flush_errors_per_handle() {
880        asked
881    } else {
882        1
883    }
884}
885
886pub(crate) fn space_err(op: Op, raw: OsError, tag: u32) -> Error {
887    match classify(Stage::Space, raw) {
888        store_io_core::errno::ErrorKind::NoSpace => Error::NoSpace {
889            tag: ReserveTag::Caller(tag),
890            ctx: ErrorContext::default(),
891        },
892        _ => io(op, raw),
893    }
894}
895
896pub(crate) fn meta_err() -> Error {
897    Error::Corruption {
898        kind: store_io_core::error::CorruptionKind::Metadata,
899        ctx: ErrorContext::default(),
900    }
901}
902
903pub(crate) fn corrupt() -> Error {
904    meta_err()
905}
906
907pub(crate) fn pair_volume(volume: VolumeId, l: &Layout) -> PairAt {
908    let (a, b) = l.volume_slots();
909    PairAt {
910        volume: *volume.as_bytes(),
911        object: VOLUME_OBJECT,
912        a,
913        b,
914        log2: l.log2_block,
915        format: l.format,
916        numbered: l.format == store_io_format::slot::FORMAT_2,
917    }
918}
919
920pub(crate) fn pair_table(volume: VolumeId, l: &Layout) -> PairAt {
921    let (a, b) = l.table_slots();
922    PairAt {
923        volume: *volume.as_bytes(),
924        object: TABLE_OBJECT,
925        a,
926        b,
927        log2: l.log2_table,
928        format: l.format,
929        numbered: false,
930    }
931}
932
933pub(crate) fn pair_region(volume: VolumeId, l: &Layout, e: &TableEntry) -> PairAt {
934    PairAt {
935        volume: *volume.as_bytes(),
936        object: l.region_object(u64::from(e.region_id)),
937        a: e.offset,
938        b: e.offset + l.block(),
939        log2: l.log2_block,
940        format: l.format,
941        numbered: false,
942    }
943}