store_io_engine/store.rs
1//! [`Store`]: one container on one device.
2//!
3//! A store owns the container file, the ownership lock, the device's flush
4//! domain, the buffer pool and the queues, and the in-memory copy of the
5//! volume record and region table. Region handles share it through an `Arc`.
6//!
7//! The metadata lock (`meta`) is taken only by lifecycle operations
8//! (provision, release, lookup by name); reads and writes of region data never
9//! take it.
10
11use std::sync::{Arc, Mutex, MutexGuard, PoisonError};
12
13pub use crate::v2::{DamagedSlot, Healed, MetadataReport, SlotDamage};
14
15use store_io_buf::{BufPool, PoolConfig};
16use store_io_core::class::{DurabilityClass, ReceiptLabel};
17use store_io_core::decide::{ClassDecision, DurableOpen, Trust};
18use store_io_core::errno::{Stage, classify};
19use store_io_core::error::{
20 Error, ErrorContext, FirstCause, Named, NotWrittenCause, Op, OsError, ReserveTag,
21};
22use store_io_core::evidence::Evidence;
23use store_io_core::id::{Generation, RegionId, VolumeId};
24use store_io_core::observe::{Observer, Snapshot};
25use store_io_format::meta::{RegionKind, RegionName, TableEntry, VolumeMeta};
26use store_io_platform::{FileName, IoOp, IoPath, LinuxTier, Platform, Queue, QueueConfig};
27
28use crate::domain::Domain;
29use crate::exec::{Lane, QueueSet, Ran, run};
30use crate::frontier::AppendFrontier;
31use crate::gate::Gate;
32use crate::layout::{Layout, TABLE_OBJECT, VOLUME_OBJECT};
33use crate::slots::{NextSlot, PairAt, PairRead, write_slot};
34use crate::space::Accounting;
35
36/// The container file name inside the store directory.
37pub const CONTAINER: &str = "store.sio";
38
39/// Explicit, reported choices for a store. Every durability-affecting field
40/// appears in [`Store::report`].
41#[derive(Debug, Clone)]
42pub struct StoreOptions {
43 /// Certificate, attestation and override inputs to the class decision.
44 pub trust: Trust,
45 /// Minimum block (slot) size in bytes; the device may require more.
46 pub block_size: Option<u32>,
47 /// Region-table slot size as a power of two (14 = 16 KiB: a format-2
48 /// table page of 169 regions; format 1's single table, 253).
49 pub log2_table_slot: u8,
50 /// Largest single I/O of the blocking layers, and the batch buffer size
51 /// (bytes; rounded up to a power of two, 128 KiB to 1 GiB).
52 pub max_io: usize,
53 /// Buffers per size class in the pool, up to `max_io` (at most
54 /// 16,777,215; more fails `create` and `open` with
55 /// `NotWritten(PoolExhausted)`).
56 pub buffers_per_class: u32,
57 /// Largest single write or read a `Submitter` takes, from one buffer, as
58 /// one OS write or read operation (bytes; rounded up to a power of two,
59 /// at most 1 GiB): the pool holds `large_buffers` buffers of that size,
60 /// used by submitters only (D-101). Ignored when not above `max_io`.
61 pub max_write: usize,
62 /// Buffers of `max_write` bytes in the pool (0: none, so a submitter's
63 /// largest operation is `max_io`; at most 16,777,215). The defaults add
64 /// 32 MiB on top of about 128 MiB of ordinary buffers per store: Windows
65 /// charges the whole pool to the commit limit at creation; on both
66 /// systems a buffer becomes resident when first used, and the most
67 /// recently returned buffer of a size is reused first, so only about as
68 /// many as are in use at once become resident. A pool too large to map
69 /// fails `create` and `open` with `NotWritten(PoolExhausted)`.
70 pub large_buffers: u32,
71 /// Queues shared by the blocking layers (0 = number of CPUs). On Linux
72 /// they perform their operations inline on either tier; the parallel
73 /// lanes, as many again and created on first use, are io_uring rings on
74 /// the io_uring tier, each holding two file descriptors (its ring and
75 /// its eventfd), as does each `Submitter`.
76 pub queues: usize,
77 /// Operations in flight per queue.
78 pub queue_depth: u32,
79 /// Worker threads for the platform's submit pool, which runs the
80 /// independent operations of a page batch or a large read concurrently
81 /// on the Linux synchronous tier (D-051, D-096); 0 = one per CPU, at most
82 /// 32. The pool is shared by every store on the platform and sized by the
83 /// first store that uses it (at most 256); other backends ignore this.
84 pub t3_pool_threads: u32,
85 /// How far, in bytes, a new append may start ahead of the oldest
86 /// incomplete one on the same region, in blocks: rounded down to a power
87 /// of two, or, on a region that cannot grow past a size smaller than the
88 /// window (format 1's regions, a capped format-2 region), that size
89 /// rounded up (so a region never limits its own appends); at least 2 and
90 /// at most 2^20 blocks. Each append region that has appended since the
91 /// open or its last recycle holds 8 bytes of memory per block of its
92 /// ring: a format-2 region that may grow (no `max_size`) holds the
93 /// window's whatever its size, 32 KiB at the default 16 MiB and 4 KiB
94 /// blocks (2 GiB for 65,536 such regions; a cap, or a smaller window,
95 /// bounds it). Default 16 MiB.
96 pub append_window: u64,
97 /// Fill new regions with the keyed incompressible pattern instead of
98 /// zeros (for thin, virtual, deduplicating or zero-detecting stacks).
99 pub keyed_fill: bool,
100 /// The kernel interface on Linux (D-082): `Auto` (io_uring where the
101 /// kernel accepts it), `Sync` or `Uring`. Read by the native API when
102 /// it makes the platform; the choice is in [`StoreReport::io`]. Other
103 /// systems ignore it.
104 pub linux_tier: LinuxTier,
105 /// Register the store's container file with each `Submitter`'s queue
106 /// where the backend can (D-078, D-107: on Linux's io_uring tier, a
107 /// read-only twin of it as a direct descriptor on the submitter's ring),
108 /// so its reads skip the per-operation file lookup in the kernel; writes
109 /// stay on the store's own descriptor, so a write in flight keeps the
110 /// store locked until it completes. Best effort: a
111 /// registration the kernel refuses is counted in `Store::io_path`, never
112 /// an error. On by default: the gates measured reads 1.03x to 1.08x of
113 /// an unregistered ring (G3d, CI); writes take the unregistered path.
114 /// Buffers
115 /// are never registered: reads into registered buffers hang forever on
116 /// some kernels (at least Linux 6.5.0 and 6.6.0), which no probe
117 /// detects (D-107).
118 pub register_file: bool,
119 /// Flushes the store runs at once (1 to 4; default 1, one in flight as
120 /// before, D-124). Several can overlap, meant for drives that execute
121 /// flushes in parallel (NVMe; not yet measured on one): each runs on a
122 /// handle and queue of its own, so each reports its own failures, and a
123 /// receipt still needs every flush up to its own to have succeeded.
124 /// Platforms that cannot keep flush failures per handle run one (Linux
125 /// 4.16 and later can); the number run is in
126 /// [`StoreReport::flushes_in_flight`]. Measure on the target drive
127 /// before raising it: at eight writers, 2 and 4 gained nothing on a
128 /// memory-backed device and lost on a virtual disk (1,314, 974 and 595
129 /// durable writes a second at 1, 2 and 4).
130 pub flushes_in_flight: u8,
131 /// Regions a format-2 store can hold, fixed at create (default 65,536:
132 /// a table of 12.1 MiB at 16 KiB table pages, written at create and
133 /// read at every open; smaller stores may ask for fewer; at most
134 /// 4,194,304). Ignored by format 1. Internal until the table's sizing
135 /// is settled (v0.5).
136 #[doc(hidden)]
137 pub region_capacity: u32,
138 /// The container format a create writes: 2 (the default). Format 1 is
139 /// for tests and migration fixtures only: a writable open of a format-1
140 /// store is refused (`Unsupported(FormatVersion)`; copy it to format 2
141 /// with `copy_store`), a read-only open proceeds.
142 #[doc(hidden)]
143 pub container_format: u16,
144 /// The options the `provision_{append,page}_region_default` calls (of
145 /// [`Store`] and [`Reservation`](crate::Reservation)) use (default
146 /// none: those calls are refused). An open-time choice, never stored:
147 /// each open may give others, and they apply to regions provisioned
148 /// from then on (an existing region keeps its size and cap). Checked
149 /// when used, so options a later call would refuse never fail an open.
150 pub region_defaults: Option<crate::RegionOptions>,
151 /// Device pacing (PLANNING R5.4): background work (scans, copies,
152 /// background `Submitter`s; paced as each lands in v0.6) is split into
153 /// pieces of this many bytes, whatever the device's state, except a
154 /// copy's fill of its target, which keeps one piece in flight: while
155 /// commits are quiet its pieces are `background_unpaced` of these, at
156 /// most 1 MiB (as many bytes as a deeper call may then keep in flight;
157 /// D-156). A multiple of
158 /// 4096 from 4096 to 256 KiB (PERF-10's piece); default 256 KiB. The
159 /// store splits at the smaller of this and its largest I/O (`max_io`
160 /// rounded up to a power of two, at least 128 KiB), rounded down to
161 /// whole blocks, at least one; [`crate::QosReport::background_io`] says
162 /// what it uses.
163 pub background_io: u32,
164 /// Background pieces in flight per background call while the device's
165 /// commits are not active: 1 to 64; default 8. Stores of one device take
166 /// the smallest.
167 pub background_unpaced: u32,
168 /// Pace background work while the device's commits are active (default
169 /// `true`): one piece in flight per background call. Off for
170 /// measurements such as G19's negative control; stores of one device
171 /// pace together if any of them asks for it.
172 pub background_pacing: bool,
173 /// How long after a commit the device's commits count as active: at
174 /// least 1 ms; default 5 s. Stores of one device take the largest.
175 pub commit_quiet: std::time::Duration,
176 /// Opt-in auto-tuning of the device's background budget (PLANNING
177 /// R5.4; default `false`). While commits are active, the pieces the
178 /// device group admits beyond each background call's own one follow
179 /// windows of commit latency (200 commits, or 2 s with at least 50),
180 /// each compared with a window run at the floor (1) in the same
181 /// conditions, not with an idle figure: the budget rises by one after
182 /// three windows within 1.25x of the latest floor window's p99, halves
183 /// at once on one over 1.4x, and every eighth window measures the floor
184 /// again, up to `background_depth_max`. Its bound, documented and not
185 /// gated: commits may run at up to 1.4x the floor's own latency for as
186 /// long as background work runs (the default, off, keeps the floor,
187 /// which G19 gates within 2x of idle). On only if every store of the
188 /// device asks for it. A device whose commits are only a power-safe
189 /// store's asynchronous `Submitter` writes is never sampled, so stays
190 /// at the floor. [`crate::QosReport`] says what it does.
191 pub background_autotune: bool,
192 /// Auto-tuning's ceiling: the background pieces the device group may
193 /// admit while commits are active (1 to 8; default 8), and never more
194 /// than `background_unpaced`. Stores of one device take the smallest.
195 pub background_depth_max: u32,
196}
197
198impl Default for StoreOptions {
199 fn default() -> Self {
200 Self {
201 trust: Trust::default(),
202 block_size: None,
203 log2_table_slot: 14,
204 max_io: 1 << 20,
205 buffers_per_class: 64,
206 max_write: 4 << 20,
207 large_buffers: 8,
208 queues: 0,
209 queue_depth: 64,
210 t3_pool_threads: 0,
211 append_window: 16 << 20,
212 keyed_fill: false,
213 linux_tier: LinuxTier::Auto,
214 register_file: true,
215 flushes_in_flight: 1,
216 region_capacity: 65_536,
217 container_format: store_io_format::slot::FORMAT_2,
218 region_defaults: None,
219 background_io: 256 << 10,
220 background_unpaced: 8,
221 background_autotune: false,
222 background_depth_max: 8,
223 background_pacing: true,
224 commit_quiet: std::time::Duration::from_secs(5),
225 }
226 }
227}
228
229/// Checks the pacing options (PLANNING R5.4): `NotWritten(InvalidOptions)`
230/// for a `background_io` that is not a multiple of 4096 from 4096 to
231/// 256 KiB, a `background_unpaced` outside 1 to 64, a
232/// `background_depth_max` outside 1 to 8, or a `commit_quiet` under 1 ms.
233pub(crate) fn check_pacing(opts: &StoreOptions) -> Result<(), Error> {
234 let io = opts.background_io as usize;
235 let ok = io >= 4096
236 && io % 4096 == 0
237 && io <= 256 << 10
238 && (1..=64).contains(&opts.background_unpaced)
239 && (1..=8).contains(&opts.background_depth_max)
240 && opts.commit_quiet >= std::time::Duration::from_millis(1);
241 if ok {
242 Ok(())
243 } else {
244 Err(Error::NotWritten {
245 cause: store_io_core::error::NotWrittenCause::InvalidOptions,
246 ctx: ErrorContext::default(),
247 })
248 }
249}
250
251/// The pacing options a store with blocks of `block` bytes joins its device
252/// group with: its piece never past its largest I/O, and whole blocks (at
253/// least one), so a piece is never a misaligned or partial-block I/O.
254pub(crate) fn qos_options(opts: &StoreOptions, block: u64) -> crate::qos::QosOptions {
255 let block = u32::try_from(block).unwrap_or(u32::MAX).max(1);
256 let piece = opts
257 .background_io
258 .min(u32::try_from(io_max(opts)).unwrap_or(u32::MAX));
259 crate::qos::QosOptions {
260 pacing: opts.background_pacing,
261 commit_quiet_ns: u64::try_from(opts.commit_quiet.as_nanos()).unwrap_or(u64::MAX),
262 background_io: (piece / block * block).max(block),
263 background_unpaced: opts.background_unpaced,
264 autotune: opts.background_autotune,
265 depth_max: opts.background_depth_max,
266 }
267}
268
269/// Joins the device group a store with `ev` and `opts` belongs to (R5.4):
270/// never fails; what the evidence does not name is paced the safe way.
271pub(crate) fn join_group<P: Platform>(
272 platform: &P,
273 ev: &Evidence,
274 opts: &StoreOptions,
275 block: u64,
276) -> Arc<crate::qos::Qos> {
277 crate::qos::Registry::global().join(
278 platform.identity_scope(),
279 &placement(ev),
280 qos_options(opts, block),
281 platform.monotonic_ns(),
282 )
283}
284
285/// Where a store with `ev` joins pacing: a memory file system joins no
286/// group; anything else by its device's keys.
287pub(crate) fn placement(ev: &Evidence) -> crate::qos::Placement {
288 if ev.fs.kind == store_io_core::evidence::FsKind::Tmpfs {
289 return crate::qos::Placement::Memory;
290 }
291 crate::qos::Placement::Keyed {
292 identity: ev.device.identity.clone(),
293 members: ev.device.members.clone(),
294 fs_device: ev.fs.device,
295 }
296}
297
298/// How durable writes are made durable on this store.
299#[derive(Debug, Clone, Copy)]
300pub(crate) struct Policy {
301 pub(crate) class: DurabilityClass,
302 pub(crate) label: ReceiptLabel,
303 /// Write with `dsync` (durable at completion): power-safe stores.
304 pub(crate) dsync: bool,
305 /// Barriers issue a device flush: every class except power-safe (where
306 /// only regions forcing data sync flush, D-102).
307 pub(crate) flush: bool,
308}
309
310pub use crate::report::{Repair, Unavailable, UnavailableRegion};
311
312/// What the store learned about its device, and what it decided.
313#[derive(Debug, Clone)]
314#[non_exhaustive]
315pub struct StoreReport {
316 /// Raw evidence from the platform.
317 pub evidence: Evidence,
318 /// The class decision.
319 pub decision: ClassDecision,
320 /// Block (slot) size in bytes.
321 pub block_size: u64,
322 /// Region-table capacity.
323 pub table_capacity: usize,
324 /// The volume identity.
325 pub volume: VolumeId,
326 /// Owner generation of this open (0 for read-only).
327 pub owner_generation: u64,
328 /// Whether the store was opened read-only.
329 pub read_only: bool,
330 /// The kernel interface the store's I/O uses, and why, as of create or
331 /// open. Queues made later (a `Submitter`'s, the parallel lanes, created
332 /// on first use) are counted live in [`Store::io_path`].
333 pub io: IoPath,
334 /// Regions the region table names that could not be opened, with the
335 /// reason (D-090), as the open found them (repairs since:
336 /// `Store::unavailable_regions`). Empty for a healthy store.
337 pub unavailable: Vec<UnavailableRegion>,
338 /// The container format (1 or 2).
339 pub format: u16,
340 /// Format 2: what the open found in the metadata (empty for format 1).
341 pub metadata: MetadataReport,
342 /// Flushes run at once ([`StoreOptions::flushes_in_flight`] as far as
343 /// the store and platform allow): 1 for a read-only store, a device that
344 /// needs no flush, or a platform that cannot keep flush failures per
345 /// handle; fewer where a handle for a further slot could not be made.
346 pub flushes_in_flight: u8,
347}
348
349/// In-memory metadata, guarded by the metadata lock.
350pub(crate) struct Meta {
351 pub(crate) volume: VolumeMeta,
352 pub(crate) volume_pair: PairRead,
353 pub(crate) table_pair: PairRead,
354 pub(crate) next_region_id: u32,
355 pub(crate) entries: Vec<TableEntry>,
356 pub(crate) container_len: u64,
357 /// Format 2's metadata (the format-1 fields above are then unused).
358 pub(crate) v2: Option<Box<crate::v2::Meta2>>,
359 /// The regions the table names that cannot be used, as they stand now
360 /// (the open's list, less those repaired since): lookups, the names
361 /// taken and growth read this, never the open-time report.
362 pub(crate) unavailable: Vec<UnavailableRegion>,
363 /// What the store's repairs did, in order ([`Store::repairs`]).
364 pub(crate) repairs: Vec<crate::report::Repair>,
365}
366
367/// One live region.
368pub(crate) struct LiveRegion {
369 pub(crate) id: RegionId,
370 pub(crate) name: RegionName,
371 pub(crate) kind: RegionKind,
372 pub(crate) generation: Generation,
373 /// Where the data lies in the container (every file offset of the
374 /// region's data comes from it).
375 /// Its length is the region's data size.
376 pub(crate) map: crate::regionmap::RegionMap,
377 /// The header pair's slot offsets.
378 pub(crate) header_at: [u64; 2],
379 pub(crate) header: Mutex<PairRead>,
380 pub(crate) frontier: Option<AppendFrontier>,
381 /// Append regions: whether the append position is known (true when
382 /// provisioned in this process; after a reopen, only after `resume_at`).
383 pub(crate) positioned: std::sync::atomic::AtomicBool,
384 /// Admission of data operations; closed by recycle and release.
385 pub(crate) gate: Gate,
386 /// Every barrier on the region issues a data flush, even on a
387 /// power-safe store (D-102). Runtime only: never on disk. Shared by
388 /// every generation of the region, so a setting made through any handle
389 /// holds after a recycle.
390 pub(crate) force_sync: Arc<std::sync::atomic::AtomicBool>,
391 /// Writes refused, the store being writable: format 2's regions on a
392 /// page that is not confirmed, possibly superseded, or with a header
393 /// feature this version may not write (always false in format 1).
394 /// Lifted by `accept_page_rollback` once every page is confirmed, never
395 /// for a superseded or foreign one.
396 pub(crate) read_only: std::sync::atomic::AtomicBool,
397 /// Possibly superseded (spec §9 step 4): another region's newer
398 /// footprint may cover its bytes; it stays read-only for good.
399 pub(crate) superseded: bool,
400 /// Its header's winner carries a read-only-compatible feature this
401 /// version does not know (spec §14): read-only for good too.
402 pub(crate) foreign: bool,
403}
404
405impl LiveRegion {
406 /// Whether barriers on this region always flush (D-102).
407 pub(crate) fn forced(&self) -> bool {
408 self.force_sync.load(std::sync::atomic::Ordering::Acquire)
409 }
410
411 /// Whether writes to the region are refused (see `read_only`).
412 pub(crate) fn is_read_only(&self) -> bool {
413 self.read_only.load(std::sync::atomic::Ordering::Acquire)
414 }
415}
416
417pub(crate) struct Inner<P: Platform> {
418 /// Declared first, so the queues drop first (fields drop in declaration
419 /// order): a queue whose operations run on worker threads waits for them
420 /// in its drop, and none may land after the ownership lock is released.
421 pub(crate) queues: QueueSet<P::Queue>,
422 /// The flush thread (D-099), started by the first submitter request
423 /// that needs a flush (a flush-required store, or a region forcing data
424 /// sync); `None` inside if the OS refused it. Declared
425 /// early too: it is stopped and joined before the file and the ownership
426 /// lock are released.
427 pub(crate) flusher: std::sync::OnceLock<Option<crate::flusher::Flusher<P>>>,
428 pub(crate) platform: P,
429 /// The store directory, held open for directory syncs.
430 pub(crate) _dir: P::Dir,
431 pub(crate) file: P::File,
432 /// Flush slots 1.. (D-124): a handle each, opened on the container
433 /// beside `file` (slot 0's). Empty with one slot.
434 pub(crate) flush_files: Box<[P::File]>,
435 pub(crate) _lock: P::Lock,
436 pub(crate) volume: VolumeId,
437 pub(crate) layout: Layout,
438 pub(crate) policy: Policy,
439 pub(crate) domain: Domain,
440 /// The store's device group (PLANNING R5.4); the domain stamps its
441 /// commit activity.
442 pub(crate) qos: Arc<crate::qos::Qos>,
443 pub(crate) pool: BufPool,
444 pub(crate) meta: Mutex<Meta>,
445 pub(crate) regions: Mutex<Vec<Arc<LiveRegion>>>,
446 /// What the open found (never changed after: see `Meta::unavailable`
447 /// and `frozen` for what stands now).
448 pub(crate) report: StoreReport,
449 /// Allocation is frozen (format 2: a page is Unconfirmed or Lost), as
450 /// it stands now: set at open, only ever cleared afterwards (once no
451 /// page is stale), so a check made without the metadata lock can only
452 /// refuse what it might have allowed, never the reverse.
453 pub(crate) frozen: std::sync::atomic::AtomicBool,
454 pub(crate) read_only: bool,
455 pub(crate) owner_generation: u64,
456 pub(crate) opts: StoreOptions,
457 /// The largest ordinary pool buffer (`Store::ordinary`), fixed at open.
458 pub(crate) ordinary: usize,
459 /// One write's largest piece (`Store::chunk`): the ordinary buffer size,
460 /// in whole blocks; fixed at open.
461 pub(crate) chunk: usize,
462 /// Space accounting (lock order: `meta`, then `acct`).
463 pub(crate) acct: Mutex<Accounting>,
464 /// Unique per open store in this process: tickets and receipts of one
465 /// open never vouch for another's writes.
466 pub(crate) epoch: u64,
467 /// Per-stage timing and counters (zero-sized without feature `observe`).
468 pub(crate) observer: Observer,
469 /// What `oldest_outstanding` queries have seen (D-095).
470 pub(crate) seen: Mutex<crate::stats::Seen>,
471 /// Live submitters' busy marks (D-098).
472 pub(crate) submitters: Mutex<crate::stats::Submitters>,
473}
474
475/// A store: one container file on one device.
476pub struct Store<P: Platform> {
477 pub(crate) inner: Arc<Inner<P>>,
478}
479
480impl<P: Platform> core::fmt::Debug for Store<P> {
481 fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
482 f.debug_struct("Store")
483 .field("volume", &self.inner.volume)
484 .field("class", &self.inner.policy.class)
485 .field("read_only", &self.inner.read_only)
486 .field("poisoned", &self.inner.domain.poisoned().is_some())
487 .finish()
488 }
489}
490
491impl<P: Platform> Clone for Store<P> {
492 fn clone(&self) -> Self {
493 Self {
494 inner: Arc::clone(&self.inner),
495 }
496 }
497}
498
499pub(crate) fn io(op: Op, raw: OsError) -> Error {
500 Error::Io {
501 op,
502 raw,
503 ctx: ErrorContext::default(),
504 }
505}
506
507pub(crate) fn lock_meta(m: &Mutex<Meta>) -> MutexGuard<'_, Meta> {
508 m.lock().unwrap_or_else(PoisonError::into_inner)
509}
510
511/// A random 128-bit identity. Uniqueness matters, secrecy does not: the
512/// standard library's OS-seeded hasher keys, mixed with the clock and the
513/// process id through SplitMix64.
514pub(crate) fn random_uuid() -> [u8; 16] {
515 use std::hash::{BuildHasher, Hasher};
516 let mut h = std::collections::hash_map::RandomState::new().build_hasher();
517 h.write_u128(
518 std::time::SystemTime::now()
519 .duration_since(std::time::UNIX_EPOCH)
520 .map_or(0, |d| d.as_nanos()),
521 );
522 h.write_u32(std::process::id());
523 let mut z = h.finish();
524 let mut out = [0u8; 16];
525 for chunk in out.chunks_exact_mut(8) {
526 z = z.wrapping_add(0x9E37_79B9_7F4A_7C15);
527 let mut x = z;
528 x = (x ^ (x >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
529 x = (x ^ (x >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
530 chunk.copy_from_slice(&(x ^ (x >> 31)).to_le_bytes());
531 }
532 out
533}
534
535impl<P: Platform> Store<P>
536where
537 P::Queue: Send,
538{
539 /// The failure that poisoned the store, if one has: every write and
540 /// barrier fails with `Poisoned` from then on. A cheap check, for a
541 /// health probe or a monitor.
542 #[must_use]
543 pub fn poisoned(&self) -> Option<FirstCause> {
544 self.inner.domain.poisoned()
545 }
546
547 /// Event counters and sampled per-stage latencies since this handle was
548 /// opened. With feature `observe` off (the default) nothing is recorded
549 /// and the snapshot is all zeros.
550 #[must_use]
551 pub fn observe(&self) -> Snapshot {
552 self.inner.observer.snapshot()
553 }
554
555 /// What the store learned about its device and what it decided, as
556 /// the open found it (never changed after: what repairs change since
557 /// is in [`Self::unavailable_regions`], [`Self::frozen`] and
558 /// [`Self::repairs`]).
559 #[must_use]
560 pub fn report(&self) -> &StoreReport {
561 &self.inner.report
562 }
563
564 /// Keeps this store's device group's commits active until the hold
565 /// drops (PLANNING R5.4), so background work on the device (scans,
566 /// copies, background `Submitter`s, of this store or another on the
567 /// device) stays paced at one piece in flight per call. For a layer
568 /// that commits rarely but needs every commit fast: it holds one while
569 /// its transactions are open and drops it when idle. The cost: while a
570 /// hold lives, background work runs at the paced rate.
571 pub fn commit_hold(&self) -> crate::CommitHold {
572 self.inner.qos.hold()
573 }
574
575 /// This store's device pacing now: whether pacing is on, whether the
576 /// device's commits are active, the budget, the background pieces in
577 /// flight in the device group.
578 #[must_use]
579 pub fn qos(&self) -> crate::QosReport {
580 self.inner.qos.report(self.inner.platform.monotonic_ns())
581 }
582
583 /// The kernel interface the store's I/O uses now, and why: as
584 /// [`StoreReport::io`], with every queue made since counted (a ring the
585 /// kernel refused for a `Submitter` or a parallel lane, and what
586 /// served instead).
587 #[must_use]
588 pub fn io_path(&self) -> IoPath {
589 self.inner.platform.io_path()
590 }
591
592 /// The volume identity.
593 #[must_use]
594 pub fn volume(&self) -> VolumeId {
595 self.inner.volume
596 }
597
598 /// Poisons the device domain (for a caller's watchdog that decided the
599 /// device is hung). Every later write and barrier fails with
600 /// [`Error::Poisoned`]; reads keep working.
601 pub fn poison(&self) {
602 self.inner.domain.poison(FirstCause {
603 op: None,
604 raw: None,
605 });
606 for r in self.regions_snapshot() {
607 if let Some(f) = &r.frontier {
608 f.wake_all();
609 }
610 }
611 }
612
613 /// Flush-sharing counters of the device domain.
614 #[must_use]
615 pub fn domain_stats(&self) -> crate::domain::DomainStats {
616 self.inner.domain.stats()
617 }
618
619 /// Worker threads in the submit pool that runs this store's page batches
620 /// and large reads concurrently, once this store's first such call has
621 /// started its parallel queues (the pool may have been started earlier by
622 /// another store on the same platform, and stays `None` if they could
623 /// not be started);
624 /// `None` before that, or on a backend that needs no pool (IOCP, the
625 /// simulator).
626 #[must_use]
627 pub fn submit_pool_threads(&self) -> Option<usize> {
628 self.inner.queues.parallel_workers()
629 }
630
631 pub(crate) fn regions_snapshot(&self) -> Vec<Arc<LiveRegion>> {
632 self.inner
633 .regions
634 .lock()
635 .unwrap_or_else(PoisonError::into_inner)
636 .clone()
637 }
638
639 /// Names and kinds of the ready regions.
640 #[must_use]
641 pub fn regions(&self) -> Vec<(String, RegionKind)> {
642 self.regions_snapshot()
643 .iter()
644 .map(|r| (r.name.as_str().to_owned(), r.kind))
645 .collect()
646 }
647
648 pub(crate) fn find(
649 &self,
650 name: &str,
651 kind: RegionKind,
652 what: Named,
653 ) -> Result<Arc<LiveRegion>, Error> {
654 let name = RegionName::new(name).map_err(|_| Error::NotWritten {
655 cause: NotWrittenCause::InvalidName,
656 ctx: ErrorContext::default(),
657 })?;
658 if let Some(r) = self
659 .regions_snapshot()
660 .into_iter()
661 .find(|r| r.name == name && r.kind == kind)
662 {
663 return Ok(r);
664 }
665 // A region the table names but open could not use is corrupt, not
666 // missing: a caller must never mistake it for a fresh start (D-090).
667 match lock_meta(&self.inner.meta)
668 .unavailable
669 .iter()
670 .find(|u| u.name == name.as_str() && u.kind == kind)
671 .cloned()
672 {
673 Some(u) => Err(Error::Corruption {
674 kind: u.reason.corruption_kind(),
675 ctx: ErrorContext {
676 region: Some(u.id),
677 range: None,
678 },
679 }),
680 None => Err(Error::NotFound { what }),
681 }
682 }
683
684 /// Regions the region table names that open could not use, with the
685 /// reason (D-090), less any repaired since. Empty for a healthy store.
686 /// Their names stay reserved.
687 #[must_use]
688 pub fn unavailable_regions(&self) -> Vec<UnavailableRegion> {
689 lock_meta(&self.inner.meta).unavailable.clone()
690 }
691
692 /// Whether allocation is frozen now (format 2: a table page is stale
693 /// or lost; spec §9 step 2): no provisioning, no growth beyond
694 /// capacity, no release, no repair. `report().metadata.frozen` is what
695 /// the open found.
696 #[must_use]
697 pub fn frozen(&self) -> bool {
698 self.inner.frozen.load(std::sync::atomic::Ordering::Acquire)
699 }
700
701 /// What this handle's repairs and accepts did, in order
702 /// ([`Self::repair_unavailable`], [`Self::accept_page_rollback`]): the
703 /// store's log of them, for the caller to record. The library prints
704 /// nothing.
705 #[must_use]
706 pub fn repairs(&self) -> Vec<crate::report::Repair> {
707 lock_meta(&self.inner.meta).repairs.clone()
708 }
709}
710
711/// Writes one slot durably according to the store's policy: a `dsync` write
712/// on power-safe devices; a plain write plus a domain barrier otherwise.
713/// Returns the pair's new state, derived from `prev` and the bytes written;
714/// the device is never read back (D-069), so once the write is durable no
715/// later step can fail or report the old version as current.
716// The pair (location, previous state), the slot to write and the store's I/O
717// context are all needed and none derives from another.
718#[allow(clippy::too_many_arguments)]
719pub(crate) fn durable_slot<P: Platform>(
720 q: &mut Lane<P::Queue>,
721 file: &P::File,
722 pool: &BufPool,
723 at: &PairAt,
724 prev: &PairRead,
725 next: &NextSlot<'_>,
726 policy: &Policy,
727 domain: &Domain,
728) -> Result<PairRead, Error> {
729 if let Some(first) = domain.poisoned() {
730 return Err(Error::Poisoned { first });
731 }
732 // On a power-safe store the `dsync` write is the slot's flush: a
733 // commit's marks the device's commits active (a background call's
734 // marks nothing: `Domain::note_commit`).
735 let entry = if policy.dsync {
736 domain.note_commit()
737 } else {
738 None
739 };
740 let written =
741 write_slot(q, file, pool, at, next, policy.dsync, domain.pace()).inspect_err(|e| {
742 if matches!(e, Error::DurabilityUnknown { .. }) {
743 domain.poison(FirstCause {
744 op: Some(Op::Write),
745 raw: None,
746 });
747 }
748 })?;
749 // The power-safe slot write was the commit: auto-tuning's sample.
750 domain.sample_since(entry);
751 if policy.flush {
752 let need = domain.ticket();
753 // Through this lane and the main handle: slot 0's.
754 domain.barrier_main(need, || flush_data(q, file))?;
755 }
756 Ok(prev.after(written))
757}
758
759/// One device flush through a queue. A flush refused without an OS error
760/// (a full queue) carries no code: none is invented.
761pub(crate) fn flush_data<Q: Queue>(q: &mut Lane<Q>, file: &Q::File) -> Result<(), Option<OsError>> {
762 match run(q, IoOp::FlushData { file }) {
763 Ok(Ran::Done { result, .. }) => result.map(|_| ()).map_err(Some),
764 Ok(Ran::Abandoned(raw)) => Err(Some(raw)),
765 Err((_b, raw)) => Err(raw),
766 }
767}
768
769pub(crate) fn container_name() -> Result<FileName, Error> {
770 FileName::new(CONTAINER).map_err(|_| Error::NotWritten {
771 cause: NotWrittenCause::InvalidName,
772 ctx: ErrorContext::default(),
773 })
774}
775
776pub(crate) fn refuse(d: &ClassDecision) -> Result<(), Error> {
777 match d.durable_open {
778 DurableOpen::Allowed | DurableOpen::Overridden => Ok(()),
779 DurableOpen::RefusedUnsafe => Err(Error::UnsafeDevice { reasons: d.reasons }),
780 DurableOpen::RefusedUnverified => Err(Error::Unverified {
781 missing: d.missing,
782 reasons: d.reasons,
783 }),
784 }
785}
786
787pub(crate) fn policy_of(d: &ClassDecision) -> Policy {
788 let power_safe = d.class == DurabilityClass::PowerSafe;
789 Policy {
790 class: d.class,
791 label: d.label,
792 dsync: power_safe,
793 flush: !power_safe,
794 }
795}
796
797pub(crate) fn make_pool(
798 ev: &Evidence,
799 layout: &Layout,
800 opts: &StoreOptions,
801 large: bool,
802) -> Result<BufPool, Error> {
803 let align = [
804 4096usize,
805 ev.fs.dio_mem_align as usize,
806 ev.device.logical_block as usize,
807 ]
808 .into_iter()
809 .max()
810 .unwrap_or(4096)
811 .next_power_of_two();
812 let min = (layout.block() as usize).max(align);
813 let max = io_max(opts).max(min);
814 let mut cfg = PoolConfig::uniform(align, min, max, opts.buffers_per_class.max(4));
815 // One class of larger buffers for single large submitter operations
816 // (D-101), fewer of them; a request between the two sizes takes one.
817 // Never in the short-lived pools that read the metadata at open.
818 let size = large_size(opts);
819 if large && opts.large_buffers > 0 && size > max {
820 cfg.classes.push(store_io_buf::ClassSpec {
821 size,
822 count: opts.large_buffers,
823 });
824 }
825 BufPool::new(&cfg).map_err(|_| Error::NotWritten {
826 cause: NotWrittenCause::PoolExhausted,
827 ctx: ErrorContext::default(),
828 })
829}
830
831/// The largest single I/O anything may ask for: 1 GiB (above 2 GiB a single
832/// `pwrite` transfers short, above 4 GiB `WriteFile` refuses).
833const LARGEST_IO: usize = 1 << 30;
834
835/// The largest ordinary pool buffer, which the blocking layers never exceed:
836/// `max_io` rounded up to a power of two, between 128 KiB and 1 GiB.
837pub(crate) fn io_max(opts: &StoreOptions) -> usize {
838 opts.max_io.clamp(2 << 16, LARGEST_IO).next_power_of_two()
839}
840
841/// The large class's buffer size: `max_write` rounded up to a power of two,
842/// at most 1 GiB (no large class when that is not above [`io_max`]).
843pub(crate) fn large_size(opts: &StoreOptions) -> usize {
844 opts.max_write.clamp(1, LARGEST_IO).next_power_of_two()
845}
846
847pub(crate) fn make_queues<P: Platform>(
848 platform: &P,
849 opts: &StoreOptions,
850 flush_lanes: usize,
851) -> Result<QueueSet<P::Queue>, Error> {
852 let n = if opts.queues == 0 {
853 std::thread::available_parallelism().map_or(4, std::num::NonZeroUsize::get)
854 } else {
855 opts.queues
856 };
857 let cfg = QueueConfig {
858 depth: opts.queue_depth.max(1),
859 ..QueueConfig::default()
860 };
861 let mut qs = Vec::with_capacity(n);
862 for _ in 0..n {
863 qs.push(platform.queue(cfg).map_err(|e| io(Op::Queue, e))?);
864 }
865 // One more queue per flush slot, used only by that slot's flush leader
866 // (D-070, D-124).
867 let mut flush = Vec::with_capacity(flush_lanes);
868 for _ in 0..flush_lanes.max(1) {
869 flush.push(platform.queue(cfg).map_err(|e| io(Op::Queue, e))?);
870 }
871 Ok(QueueSet::with_flush_lanes(qs, flush, cfg.depth as usize))
872}
873
874/// The flush slots a store runs (D-124): `opts.flushes_in_flight` clamped to
875/// `1..=MAX_FLUSH_SLOTS`, and 1 unless the platform reports flush failures
876/// per handle.
877pub(crate) fn flush_slots<P: Platform>(platform: &P, opts: &StoreOptions) -> usize {
878 let asked = usize::from(opts.flushes_in_flight).clamp(1, crate::domain::MAX_FLUSH_SLOTS);
879 if asked > 1 && platform.flush_errors_per_handle() {
880 asked
881 } else {
882 1
883 }
884}
885
886pub(crate) fn space_err(op: Op, raw: OsError, tag: u32) -> Error {
887 match classify(Stage::Space, raw) {
888 store_io_core::errno::ErrorKind::NoSpace => Error::NoSpace {
889 tag: ReserveTag::Caller(tag),
890 ctx: ErrorContext::default(),
891 },
892 _ => io(op, raw),
893 }
894}
895
896pub(crate) fn meta_err() -> Error {
897 Error::Corruption {
898 kind: store_io_core::error::CorruptionKind::Metadata,
899 ctx: ErrorContext::default(),
900 }
901}
902
903pub(crate) fn corrupt() -> Error {
904 meta_err()
905}
906
907pub(crate) fn pair_volume(volume: VolumeId, l: &Layout) -> PairAt {
908 let (a, b) = l.volume_slots();
909 PairAt {
910 volume: *volume.as_bytes(),
911 object: VOLUME_OBJECT,
912 a,
913 b,
914 log2: l.log2_block,
915 format: l.format,
916 numbered: l.format == store_io_format::slot::FORMAT_2,
917 }
918}
919
920pub(crate) fn pair_table(volume: VolumeId, l: &Layout) -> PairAt {
921 let (a, b) = l.table_slots();
922 PairAt {
923 volume: *volume.as_bytes(),
924 object: TABLE_OBJECT,
925 a,
926 b,
927 log2: l.log2_table,
928 format: l.format,
929 numbered: false,
930 }
931}
932
933pub(crate) fn pair_region(volume: VolumeId, l: &Layout, e: &TableEntry) -> PairAt {
934 PairAt {
935 volume: *volume.as_bytes(),
936 object: l.region_object(u64::from(e.region_id)),
937 a: e.offset,
938 b: e.offset + l.block(),
939 log2: l.log2_block,
940 format: l.format,
941 numbered: false,
942 }
943}