store_io_engine/store.rs
1//! [`Store`]: one container on one device.
2//!
3//! A store owns the container file, the ownership lock, the device's flush
4//! domain, the buffer pool and the queues, and the in-memory copy of the
5//! volume record and region table. Region handles share it through an `Arc`.
6//!
7//! The metadata lock (`meta`) is taken only by lifecycle operations
8//! (provision, release, lookup by name); reads and writes of region data never
9//! take it.
10
11use std::sync::{Arc, Mutex, MutexGuard, PoisonError};
12
13pub use crate::v2::{DamagedSlot, Healed, MetadataReport, SlotDamage};
14
15use store_io_buf::{BufPool, PoolConfig};
16use store_io_core::class::{DurabilityClass, ReceiptLabel};
17use store_io_core::decide::{ClassDecision, DurableOpen, Trust};
18use store_io_core::errno::{Stage, classify};
19use store_io_core::error::{
20 Error, ErrorContext, FirstCause, Named, NotWrittenCause, Op, OsError, ReserveTag,
21};
22use store_io_core::evidence::Evidence;
23use store_io_core::id::{Generation, RegionId, VolumeId};
24use store_io_core::observe::{Observer, Snapshot};
25use store_io_format::meta::{RegionKind, RegionName, TableEntry, VolumeMeta};
26use store_io_platform::{FileName, IoOp, IoPath, LinuxTier, Platform, Queue, QueueConfig};
27
28use crate::domain::Domain;
29use crate::exec::{Lane, QueueSet, Ran, run};
30use crate::frontier::AppendFrontier;
31use crate::gate::Gate;
32use crate::layout::{Layout, TABLE_OBJECT, VOLUME_OBJECT};
33use crate::slots::{NextSlot, PairAt, PairRead, write_slot};
34use crate::space::Accounting;
35
36/// The container file name inside the store directory.
37pub const CONTAINER: &str = "store.sio";
38
39/// Explicit, reported choices for a store. Every durability-affecting field
40/// appears in [`Store::report`].
41#[derive(Debug, Clone)]
42pub struct StoreOptions {
43 /// Certificate, attestation and override inputs to the class decision.
44 pub trust: Trust,
45 /// Minimum block (slot) size in bytes; the device may require more.
46 pub block_size: Option<u32>,
47 /// Region-table slot size as a power of two (14 = 16 KiB: a format-2
48 /// table page of 169 regions; format 1's single table, 253).
49 pub log2_table_slot: u8,
50 /// Largest single I/O of the blocking layers, and the batch buffer size
51 /// (bytes; rounded up to a power of two, 128 KiB to 1 GiB).
52 pub max_io: usize,
53 /// Buffers per size class in the pool, up to `max_io` (at most
54 /// 16,777,215; more fails `create` and `open` with
55 /// `NotWritten(PoolExhausted)`).
56 pub buffers_per_class: u32,
57 /// Largest single write or read a `Submitter` takes, from one buffer, as
58 /// one OS write or read operation (bytes; rounded up to a power of two,
59 /// at most 1 GiB): the pool holds `large_buffers` buffers of that size,
60 /// used by submitters only (D-101). Ignored when not above `max_io`.
61 pub max_write: usize,
62 /// Buffers of `max_write` bytes in the pool (0: none, so a submitter's
63 /// largest operation is `max_io`; at most 16,777,215). The defaults add
64 /// 32 MiB on top of about 128 MiB of ordinary buffers per store: Windows
65 /// charges the whole pool to the commit limit at creation; on both
66 /// systems a buffer becomes resident when first used, and the most
67 /// recently returned buffer of a size is reused first, so only about as
68 /// many as are in use at once become resident. A pool too large to map
69 /// fails `create` and `open` with `NotWritten(PoolExhausted)`.
70 pub large_buffers: u32,
71 /// Queues shared by the blocking layers (0 = number of CPUs). On Linux
72 /// they perform their operations inline on either tier; the parallel
73 /// lanes, as many again and created on first use, are io_uring rings on
74 /// the io_uring tier, each holding two file descriptors (its ring and
75 /// its eventfd), as does each `Submitter`.
76 pub queues: usize,
77 /// Operations in flight per queue.
78 pub queue_depth: u32,
79 /// Worker threads for the platform's submit pool, which runs the
80 /// independent operations of a page batch or a large read concurrently
81 /// on the Linux synchronous tier (D-051, D-096); 0 = one per CPU, at most
82 /// 32. The pool is shared by every store on the platform and sized by the
83 /// first store that uses it (at most 256); other backends ignore this.
84 pub t3_pool_threads: u32,
85 /// How far, in bytes, a new append may start ahead of the oldest
86 /// incomplete one on the same region, in blocks: rounded down to a power
87 /// of two, or, on a region that cannot grow past a size smaller than the
88 /// window (format 1's regions, a capped format-2 region), that size
89 /// rounded up (so a region never limits its own appends); at least 2 and
90 /// at most 2^20 blocks. Each append region that has appended since the
91 /// open or its last recycle holds 8 bytes of memory per block of its
92 /// ring: a format-2 region that may grow (no `max_size`) holds the
93 /// window's whatever its size, 32 KiB at the default 16 MiB and 4 KiB
94 /// blocks (2 GiB for 65,536 such regions; a cap, or a smaller window,
95 /// bounds it). Default 16 MiB.
96 pub append_window: u64,
97 /// Fill new regions with the keyed incompressible pattern instead of
98 /// zeros (for thin, virtual, deduplicating or zero-detecting stacks).
99 pub keyed_fill: bool,
100 /// The kernel interface on Linux (D-082): `Auto` (io_uring where the
101 /// kernel accepts it), `Sync` or `Uring`. Read by the native API when
102 /// it makes the platform; the choice is in [`StoreReport::io`]. Other
103 /// systems ignore it.
104 pub linux_tier: LinuxTier,
105 /// Register the store's container file with each `Submitter`'s queue
106 /// where the backend can (D-078, D-107: on Linux's io_uring tier, a
107 /// read-only twin of it as a direct descriptor on the submitter's ring),
108 /// so its reads skip the per-operation file lookup in the kernel; writes
109 /// stay on the store's own descriptor, so a write in flight keeps the
110 /// store locked until it completes. Best effort: a
111 /// registration the kernel refuses is counted in `Store::io_path`, never
112 /// an error. On by default: the gates measured reads 1.03x to 1.08x of
113 /// an unregistered ring (G3d, CI); writes take the unregistered path.
114 /// Buffers
115 /// are never registered: reads into registered buffers hang forever on
116 /// some kernels (at least Linux 6.5.0 and 6.6.0), which no probe
117 /// detects (D-107).
118 pub register_file: bool,
119 /// Flushes the store runs at once (1 to 4; default 1, one in flight as
120 /// before, D-124). Several can overlap, meant for drives that execute
121 /// flushes in parallel (NVMe; not yet measured on one): each runs on a
122 /// handle and queue of its own, so each reports its own failures, and a
123 /// receipt still needs every flush up to its own to have succeeded.
124 /// Platforms that cannot keep flush failures per handle run one (Linux
125 /// 4.16 and later can); the number run is in
126 /// [`StoreReport::flushes_in_flight`]. Measure on the target drive
127 /// before raising it: at eight writers, 2 and 4 gained nothing on a
128 /// memory-backed device and lost on a virtual disk (1,314, 974 and 595
129 /// durable writes a second at 1, 2 and 4).
130 pub flushes_in_flight: u8,
131 /// Regions a format-2 store can hold, fixed at create (default 65,536:
132 /// a table of 12.1 MiB at 16 KiB table pages, written at create and
133 /// read at every open; smaller stores may ask for fewer; at most
134 /// 4,194,304). Ignored by format 1. Internal until the table's sizing
135 /// is settled (v0.5).
136 #[doc(hidden)]
137 pub region_capacity: u32,
138 /// The container format a create writes: 2 (the default). Format 1 is
139 /// for tests and migration fixtures only: a writable open of a format-1
140 /// store is refused (`Unsupported(FormatVersion)`; copy it to format 2
141 /// with `copy_store`), a read-only open proceeds.
142 #[doc(hidden)]
143 pub container_format: u16,
144 /// The options the `provision_{append,page}_region_default` calls (of
145 /// [`Store`] and [`Reservation`](crate::Reservation)) use (default
146 /// none: those calls are refused). An open-time choice, never stored:
147 /// each open may give others, and they apply to regions provisioned
148 /// from then on (an existing region keeps its size and cap). Checked
149 /// when used, so options a later call would refuse never fail an open.
150 pub region_defaults: Option<crate::RegionOptions>,
151}
152
153impl Default for StoreOptions {
154 fn default() -> Self {
155 Self {
156 trust: Trust::default(),
157 block_size: None,
158 log2_table_slot: 14,
159 max_io: 1 << 20,
160 buffers_per_class: 64,
161 max_write: 4 << 20,
162 large_buffers: 8,
163 queues: 0,
164 queue_depth: 64,
165 t3_pool_threads: 0,
166 append_window: 16 << 20,
167 keyed_fill: false,
168 linux_tier: LinuxTier::Auto,
169 register_file: true,
170 flushes_in_flight: 1,
171 region_capacity: 65_536,
172 container_format: store_io_format::slot::FORMAT_2,
173 region_defaults: None,
174 }
175 }
176}
177
178/// How durable writes are made durable on this store.
179#[derive(Debug, Clone, Copy)]
180pub(crate) struct Policy {
181 pub(crate) class: DurabilityClass,
182 pub(crate) label: ReceiptLabel,
183 /// Write with `dsync` (durable at completion): power-safe stores.
184 pub(crate) dsync: bool,
185 /// Barriers issue a device flush: every class except power-safe (where
186 /// only regions forcing data sync flush, D-102).
187 pub(crate) flush: bool,
188}
189
190pub use crate::report::{Repair, Unavailable, UnavailableRegion};
191
192/// What the store learned about its device, and what it decided.
193#[derive(Debug, Clone)]
194#[non_exhaustive]
195pub struct StoreReport {
196 /// Raw evidence from the platform.
197 pub evidence: Evidence,
198 /// The class decision.
199 pub decision: ClassDecision,
200 /// Block (slot) size in bytes.
201 pub block_size: u64,
202 /// Region-table capacity.
203 pub table_capacity: usize,
204 /// The volume identity.
205 pub volume: VolumeId,
206 /// Owner generation of this open (0 for read-only).
207 pub owner_generation: u64,
208 /// Whether the store was opened read-only.
209 pub read_only: bool,
210 /// The kernel interface the store's I/O uses, and why, as of create or
211 /// open. Queues made later (a `Submitter`'s, the parallel lanes, created
212 /// on first use) are counted live in [`Store::io_path`].
213 pub io: IoPath,
214 /// Regions the region table names that could not be opened, with the
215 /// reason (D-090), as the open found them (repairs since:
216 /// `Store::unavailable_regions`). Empty for a healthy store.
217 pub unavailable: Vec<UnavailableRegion>,
218 /// The container format (1 or 2).
219 pub format: u16,
220 /// Format 2: what the open found in the metadata (empty for format 1).
221 pub metadata: MetadataReport,
222 /// Flushes run at once ([`StoreOptions::flushes_in_flight`] as far as
223 /// the store and platform allow): 1 for a read-only store, a device that
224 /// needs no flush, or a platform that cannot keep flush failures per
225 /// handle; fewer where a handle for a further slot could not be made.
226 pub flushes_in_flight: u8,
227}
228
229/// In-memory metadata, guarded by the metadata lock.
230pub(crate) struct Meta {
231 pub(crate) volume: VolumeMeta,
232 pub(crate) volume_pair: PairRead,
233 pub(crate) table_pair: PairRead,
234 pub(crate) next_region_id: u32,
235 pub(crate) entries: Vec<TableEntry>,
236 pub(crate) container_len: u64,
237 /// Format 2's metadata (the format-1 fields above are then unused).
238 pub(crate) v2: Option<Box<crate::v2::Meta2>>,
239 /// The regions the table names that cannot be used, as they stand now
240 /// (the open's list, less those repaired since): lookups, the names
241 /// taken and growth read this, never the open-time report.
242 pub(crate) unavailable: Vec<UnavailableRegion>,
243 /// What the store's repairs did, in order ([`Store::repairs`]).
244 pub(crate) repairs: Vec<crate::report::Repair>,
245}
246
247/// One live region.
248pub(crate) struct LiveRegion {
249 pub(crate) id: RegionId,
250 pub(crate) name: RegionName,
251 pub(crate) kind: RegionKind,
252 pub(crate) generation: Generation,
253 /// Where the data lies in the container (every file offset of the
254 /// region's data comes from it).
255 /// Its length is the region's data size.
256 pub(crate) map: crate::regionmap::RegionMap,
257 /// The header pair's slot offsets.
258 pub(crate) header_at: [u64; 2],
259 pub(crate) header: Mutex<PairRead>,
260 pub(crate) frontier: Option<AppendFrontier>,
261 /// Append regions: whether the append position is known (true when
262 /// provisioned in this process; after a reopen, only after `resume_at`).
263 pub(crate) positioned: std::sync::atomic::AtomicBool,
264 /// Admission of data operations; closed by recycle and release.
265 pub(crate) gate: Gate,
266 /// Every barrier on the region issues a data flush, even on a
267 /// power-safe store (D-102). Runtime only: never on disk. Shared by
268 /// every generation of the region, so a setting made through any handle
269 /// holds after a recycle.
270 pub(crate) force_sync: Arc<std::sync::atomic::AtomicBool>,
271 /// Writes refused, the store being writable: format 2's regions on a
272 /// page that is not confirmed, possibly superseded, or with a header
273 /// feature this version may not write (always false in format 1).
274 /// Lifted by `accept_page_rollback` once every page is confirmed, never
275 /// for a superseded or foreign one.
276 pub(crate) read_only: std::sync::atomic::AtomicBool,
277 /// Possibly superseded (spec §9 step 4): another region's newer
278 /// footprint may cover its bytes; it stays read-only for good.
279 pub(crate) superseded: bool,
280 /// Its header's winner carries a read-only-compatible feature this
281 /// version does not know (spec §14): read-only for good too.
282 pub(crate) foreign: bool,
283}
284
285impl LiveRegion {
286 /// Whether barriers on this region always flush (D-102).
287 pub(crate) fn forced(&self) -> bool {
288 self.force_sync.load(std::sync::atomic::Ordering::Acquire)
289 }
290
291 /// Whether writes to the region are refused (see `read_only`).
292 pub(crate) fn is_read_only(&self) -> bool {
293 self.read_only.load(std::sync::atomic::Ordering::Acquire)
294 }
295}
296
297pub(crate) struct Inner<P: Platform> {
298 /// Declared first, so the queues drop first (fields drop in declaration
299 /// order): a queue whose operations run on worker threads waits for them
300 /// in its drop, and none may land after the ownership lock is released.
301 pub(crate) queues: QueueSet<P::Queue>,
302 /// The flush thread (D-099), started by the first submitter request
303 /// that needs a flush (a flush-required store, or a region forcing data
304 /// sync); `None` inside if the OS refused it. Declared
305 /// early too: it is stopped and joined before the file and the ownership
306 /// lock are released.
307 pub(crate) flusher: std::sync::OnceLock<Option<crate::flusher::Flusher<P>>>,
308 pub(crate) platform: P,
309 /// The store directory, held open for directory syncs.
310 pub(crate) _dir: P::Dir,
311 pub(crate) file: P::File,
312 /// Flush slots 1.. (D-124): a handle each, opened on the container
313 /// beside `file` (slot 0's). Empty with one slot.
314 pub(crate) flush_files: Box<[P::File]>,
315 pub(crate) _lock: P::Lock,
316 pub(crate) volume: VolumeId,
317 pub(crate) layout: Layout,
318 pub(crate) policy: Policy,
319 pub(crate) domain: Domain,
320 pub(crate) pool: BufPool,
321 pub(crate) meta: Mutex<Meta>,
322 pub(crate) regions: Mutex<Vec<Arc<LiveRegion>>>,
323 /// What the open found (never changed after: see `Meta::unavailable`
324 /// and `frozen` for what stands now).
325 pub(crate) report: StoreReport,
326 /// Allocation is frozen (format 2: a page is Unconfirmed or Lost), as
327 /// it stands now: set at open, only ever cleared afterwards (once no
328 /// page is stale), so a check made without the metadata lock can only
329 /// refuse what it might have allowed, never the reverse.
330 pub(crate) frozen: std::sync::atomic::AtomicBool,
331 pub(crate) read_only: bool,
332 pub(crate) owner_generation: u64,
333 pub(crate) opts: StoreOptions,
334 /// The largest ordinary pool buffer (`Store::ordinary`), fixed at open.
335 pub(crate) ordinary: usize,
336 /// One write's largest piece (`Store::chunk`): the ordinary buffer size,
337 /// in whole blocks; fixed at open.
338 pub(crate) chunk: usize,
339 /// Space accounting (lock order: `meta`, then `acct`).
340 pub(crate) acct: Mutex<Accounting>,
341 /// Unique per open store in this process: tickets and receipts of one
342 /// open never vouch for another's writes.
343 pub(crate) epoch: u64,
344 /// Per-stage timing and counters (zero-sized without feature `observe`).
345 pub(crate) observer: Observer,
346 /// What `oldest_outstanding` queries have seen (D-095).
347 pub(crate) seen: Mutex<crate::stats::Seen>,
348 /// Live submitters' busy marks (D-098).
349 pub(crate) submitters: Mutex<crate::stats::Submitters>,
350}
351
352/// A store: one container file on one device.
353pub struct Store<P: Platform> {
354 pub(crate) inner: Arc<Inner<P>>,
355}
356
357impl<P: Platform> core::fmt::Debug for Store<P> {
358 fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
359 f.debug_struct("Store")
360 .field("volume", &self.inner.volume)
361 .field("class", &self.inner.policy.class)
362 .field("read_only", &self.inner.read_only)
363 .field("poisoned", &self.inner.domain.poisoned().is_some())
364 .finish()
365 }
366}
367
368impl<P: Platform> Clone for Store<P> {
369 fn clone(&self) -> Self {
370 Self {
371 inner: Arc::clone(&self.inner),
372 }
373 }
374}
375
376pub(crate) fn io(op: Op, raw: OsError) -> Error {
377 Error::Io {
378 op,
379 raw,
380 ctx: ErrorContext::default(),
381 }
382}
383
384pub(crate) fn lock_meta(m: &Mutex<Meta>) -> MutexGuard<'_, Meta> {
385 m.lock().unwrap_or_else(PoisonError::into_inner)
386}
387
388/// A random 128-bit identity. Uniqueness matters, secrecy does not: the
389/// standard library's OS-seeded hasher keys, mixed with the clock and the
390/// process id through SplitMix64.
391pub(crate) fn random_uuid() -> [u8; 16] {
392 use std::hash::{BuildHasher, Hasher};
393 let mut h = std::collections::hash_map::RandomState::new().build_hasher();
394 h.write_u128(
395 std::time::SystemTime::now()
396 .duration_since(std::time::UNIX_EPOCH)
397 .map_or(0, |d| d.as_nanos()),
398 );
399 h.write_u32(std::process::id());
400 let mut z = h.finish();
401 let mut out = [0u8; 16];
402 for chunk in out.chunks_exact_mut(8) {
403 z = z.wrapping_add(0x9E37_79B9_7F4A_7C15);
404 let mut x = z;
405 x = (x ^ (x >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
406 x = (x ^ (x >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
407 chunk.copy_from_slice(&(x ^ (x >> 31)).to_le_bytes());
408 }
409 out
410}
411
412impl<P: Platform> Store<P>
413where
414 P::Queue: Send,
415{
416 /// The failure that poisoned the store, if one has: every write and
417 /// barrier fails with `Poisoned` from then on. A cheap check, for a
418 /// health probe or a monitor.
419 #[must_use]
420 pub fn poisoned(&self) -> Option<FirstCause> {
421 self.inner.domain.poisoned()
422 }
423
424 /// Event counters and sampled per-stage latencies since this handle was
425 /// opened. With feature `observe` off (the default) nothing is recorded
426 /// and the snapshot is all zeros.
427 #[must_use]
428 pub fn observe(&self) -> Snapshot {
429 self.inner.observer.snapshot()
430 }
431
432 /// What the store learned about its device and what it decided, as
433 /// the open found it (never changed after: what repairs change since
434 /// is in [`Self::unavailable_regions`], [`Self::frozen`] and
435 /// [`Self::repairs`]).
436 #[must_use]
437 pub fn report(&self) -> &StoreReport {
438 &self.inner.report
439 }
440
441 /// The kernel interface the store's I/O uses now, and why: as
442 /// [`StoreReport::io`], with every queue made since counted (a ring the
443 /// kernel refused for a `Submitter` or a parallel lane, and what
444 /// served instead).
445 #[must_use]
446 pub fn io_path(&self) -> IoPath {
447 self.inner.platform.io_path()
448 }
449
450 /// The volume identity.
451 #[must_use]
452 pub fn volume(&self) -> VolumeId {
453 self.inner.volume
454 }
455
456 /// Poisons the device domain (for a caller's watchdog that decided the
457 /// device is hung). Every later write and barrier fails with
458 /// [`Error::Poisoned`]; reads keep working.
459 pub fn poison(&self) {
460 self.inner.domain.poison(FirstCause {
461 op: None,
462 raw: None,
463 });
464 for r in self.regions_snapshot() {
465 if let Some(f) = &r.frontier {
466 f.wake_all();
467 }
468 }
469 }
470
471 /// Flush-sharing counters of the device domain.
472 #[must_use]
473 pub fn domain_stats(&self) -> crate::domain::DomainStats {
474 self.inner.domain.stats()
475 }
476
477 /// Worker threads in the submit pool that runs this store's page batches
478 /// and large reads concurrently, once this store's first such call has
479 /// started its parallel queues (the pool may have been started earlier by
480 /// another store on the same platform, and stays `None` if they could
481 /// not be started);
482 /// `None` before that, or on a backend that needs no pool (IOCP, the
483 /// simulator).
484 #[must_use]
485 pub fn submit_pool_threads(&self) -> Option<usize> {
486 self.inner.queues.parallel_workers()
487 }
488
489 pub(crate) fn regions_snapshot(&self) -> Vec<Arc<LiveRegion>> {
490 self.inner
491 .regions
492 .lock()
493 .unwrap_or_else(PoisonError::into_inner)
494 .clone()
495 }
496
497 /// Names and kinds of the ready regions.
498 #[must_use]
499 pub fn regions(&self) -> Vec<(String, RegionKind)> {
500 self.regions_snapshot()
501 .iter()
502 .map(|r| (r.name.as_str().to_owned(), r.kind))
503 .collect()
504 }
505
506 pub(crate) fn find(
507 &self,
508 name: &str,
509 kind: RegionKind,
510 what: Named,
511 ) -> Result<Arc<LiveRegion>, Error> {
512 let name = RegionName::new(name).map_err(|_| Error::NotWritten {
513 cause: NotWrittenCause::InvalidName,
514 ctx: ErrorContext::default(),
515 })?;
516 if let Some(r) = self
517 .regions_snapshot()
518 .into_iter()
519 .find(|r| r.name == name && r.kind == kind)
520 {
521 return Ok(r);
522 }
523 // A region the table names but open could not use is corrupt, not
524 // missing: a caller must never mistake it for a fresh start (D-090).
525 match lock_meta(&self.inner.meta)
526 .unavailable
527 .iter()
528 .find(|u| u.name == name.as_str() && u.kind == kind)
529 .cloned()
530 {
531 Some(u) => Err(Error::Corruption {
532 kind: u.reason.corruption_kind(),
533 ctx: ErrorContext {
534 region: Some(u.id),
535 range: None,
536 },
537 }),
538 None => Err(Error::NotFound { what }),
539 }
540 }
541
542 /// Regions the region table names that open could not use, with the
543 /// reason (D-090), less any repaired since. Empty for a healthy store.
544 /// Their names stay reserved.
545 #[must_use]
546 pub fn unavailable_regions(&self) -> Vec<UnavailableRegion> {
547 lock_meta(&self.inner.meta).unavailable.clone()
548 }
549
550 /// Whether allocation is frozen now (format 2: a table page is stale
551 /// or lost; spec §9 step 2): no provisioning, no growth beyond
552 /// capacity, no release, no repair. `report().metadata.frozen` is what
553 /// the open found.
554 #[must_use]
555 pub fn frozen(&self) -> bool {
556 self.inner.frozen.load(std::sync::atomic::Ordering::Acquire)
557 }
558
559 /// What this handle's repairs and accepts did, in order
560 /// ([`Self::repair_unavailable`], [`Self::accept_page_rollback`]): the
561 /// store's log of them, for the caller to record. The library prints
562 /// nothing.
563 #[must_use]
564 pub fn repairs(&self) -> Vec<crate::report::Repair> {
565 lock_meta(&self.inner.meta).repairs.clone()
566 }
567}
568
569/// Writes one slot durably according to the store's policy: a `dsync` write
570/// on power-safe devices; a plain write plus a domain barrier otherwise.
571/// Returns the pair's new state, derived from `prev` and the bytes written;
572/// the device is never read back (D-069), so once the write is durable no
573/// later step can fail or report the old version as current.
574// The pair (location, previous state), the slot to write and the store's I/O
575// context are all needed and none derives from another.
576#[allow(clippy::too_many_arguments)]
577pub(crate) fn durable_slot<P: Platform>(
578 q: &mut Lane<P::Queue>,
579 file: &P::File,
580 pool: &BufPool,
581 at: &PairAt,
582 prev: &PairRead,
583 next: &NextSlot<'_>,
584 policy: &Policy,
585 domain: &Domain,
586) -> Result<PairRead, Error> {
587 if let Some(first) = domain.poisoned() {
588 return Err(Error::Poisoned { first });
589 }
590 let written = write_slot(q, file, pool, at, next, policy.dsync).inspect_err(|e| {
591 if matches!(e, Error::DurabilityUnknown { .. }) {
592 domain.poison(FirstCause {
593 op: Some(Op::Write),
594 raw: None,
595 });
596 }
597 })?;
598 if policy.flush {
599 let need = domain.ticket();
600 // Through this lane and the main handle: slot 0's.
601 domain.barrier_main(need, || flush_data(q, file))?;
602 }
603 Ok(prev.after(written))
604}
605
606/// One device flush through a queue. A flush refused without an OS error
607/// (a full queue) carries no code: none is invented.
608pub(crate) fn flush_data<Q: Queue>(q: &mut Lane<Q>, file: &Q::File) -> Result<(), Option<OsError>> {
609 match run(q, IoOp::FlushData { file }) {
610 Ok(Ran::Done { result, .. }) => result.map(|_| ()).map_err(Some),
611 Ok(Ran::Abandoned(raw)) => Err(Some(raw)),
612 Err((_b, raw)) => Err(raw),
613 }
614}
615
616pub(crate) fn container_name() -> Result<FileName, Error> {
617 FileName::new(CONTAINER).map_err(|_| Error::NotWritten {
618 cause: NotWrittenCause::InvalidName,
619 ctx: ErrorContext::default(),
620 })
621}
622
623pub(crate) fn refuse(d: &ClassDecision) -> Result<(), Error> {
624 match d.durable_open {
625 DurableOpen::Allowed | DurableOpen::Overridden => Ok(()),
626 DurableOpen::RefusedUnsafe => Err(Error::UnsafeDevice { reasons: d.reasons }),
627 DurableOpen::RefusedUnverified => Err(Error::Unverified {
628 missing: d.missing,
629 reasons: d.reasons,
630 }),
631 }
632}
633
634pub(crate) fn policy_of(d: &ClassDecision) -> Policy {
635 let power_safe = d.class == DurabilityClass::PowerSafe;
636 Policy {
637 class: d.class,
638 label: d.label,
639 dsync: power_safe,
640 flush: !power_safe,
641 }
642}
643
644pub(crate) fn make_pool(
645 ev: &Evidence,
646 layout: &Layout,
647 opts: &StoreOptions,
648 large: bool,
649) -> Result<BufPool, Error> {
650 let align = [
651 4096usize,
652 ev.fs.dio_mem_align as usize,
653 ev.device.logical_block as usize,
654 ]
655 .into_iter()
656 .max()
657 .unwrap_or(4096)
658 .next_power_of_two();
659 let min = (layout.block() as usize).max(align);
660 let max = io_max(opts).max(min);
661 let mut cfg = PoolConfig::uniform(align, min, max, opts.buffers_per_class.max(4));
662 // One class of larger buffers for single large submitter operations
663 // (D-101), fewer of them; a request between the two sizes takes one.
664 // Never in the short-lived pools that read the metadata at open.
665 let size = large_size(opts);
666 if large && opts.large_buffers > 0 && size > max {
667 cfg.classes.push(store_io_buf::ClassSpec {
668 size,
669 count: opts.large_buffers,
670 });
671 }
672 BufPool::new(&cfg).map_err(|_| Error::NotWritten {
673 cause: NotWrittenCause::PoolExhausted,
674 ctx: ErrorContext::default(),
675 })
676}
677
678/// The largest single I/O anything may ask for: 1 GiB (above 2 GiB a single
679/// `pwrite` transfers short, above 4 GiB `WriteFile` refuses).
680const LARGEST_IO: usize = 1 << 30;
681
682/// The largest ordinary pool buffer, which the blocking layers never exceed:
683/// `max_io` rounded up to a power of two, between 128 KiB and 1 GiB.
684pub(crate) fn io_max(opts: &StoreOptions) -> usize {
685 opts.max_io.clamp(2 << 16, LARGEST_IO).next_power_of_two()
686}
687
688/// The large class's buffer size: `max_write` rounded up to a power of two,
689/// at most 1 GiB (no large class when that is not above [`io_max`]).
690pub(crate) fn large_size(opts: &StoreOptions) -> usize {
691 opts.max_write.clamp(1, LARGEST_IO).next_power_of_two()
692}
693
694pub(crate) fn make_queues<P: Platform>(
695 platform: &P,
696 opts: &StoreOptions,
697 flush_lanes: usize,
698) -> Result<QueueSet<P::Queue>, Error> {
699 let n = if opts.queues == 0 {
700 std::thread::available_parallelism().map_or(4, std::num::NonZeroUsize::get)
701 } else {
702 opts.queues
703 };
704 let cfg = QueueConfig {
705 depth: opts.queue_depth.max(1),
706 ..QueueConfig::default()
707 };
708 let mut qs = Vec::with_capacity(n);
709 for _ in 0..n {
710 qs.push(platform.queue(cfg).map_err(|e| io(Op::Queue, e))?);
711 }
712 // One more queue per flush slot, used only by that slot's flush leader
713 // (D-070, D-124).
714 let mut flush = Vec::with_capacity(flush_lanes);
715 for _ in 0..flush_lanes.max(1) {
716 flush.push(platform.queue(cfg).map_err(|e| io(Op::Queue, e))?);
717 }
718 Ok(QueueSet::with_flush_lanes(qs, flush, cfg.depth as usize))
719}
720
721/// The flush slots a store runs (D-124): `opts.flushes_in_flight` clamped to
722/// `1..=MAX_FLUSH_SLOTS`, and 1 unless the platform reports flush failures
723/// per handle.
724pub(crate) fn flush_slots<P: Platform>(platform: &P, opts: &StoreOptions) -> usize {
725 let asked = usize::from(opts.flushes_in_flight).clamp(1, crate::domain::MAX_FLUSH_SLOTS);
726 if asked > 1 && platform.flush_errors_per_handle() {
727 asked
728 } else {
729 1
730 }
731}
732
733pub(crate) fn space_err(op: Op, raw: OsError, tag: u32) -> Error {
734 match classify(Stage::Space, raw) {
735 store_io_core::errno::ErrorKind::NoSpace => Error::NoSpace {
736 tag: ReserveTag::Caller(tag),
737 ctx: ErrorContext::default(),
738 },
739 _ => io(op, raw),
740 }
741}
742
743pub(crate) fn meta_err() -> Error {
744 Error::Corruption {
745 kind: store_io_core::error::CorruptionKind::Metadata,
746 ctx: ErrorContext::default(),
747 }
748}
749
750pub(crate) fn corrupt() -> Error {
751 meta_err()
752}
753
754pub(crate) fn pair_volume(volume: VolumeId, l: &Layout) -> PairAt {
755 let (a, b) = l.volume_slots();
756 PairAt {
757 volume: *volume.as_bytes(),
758 object: VOLUME_OBJECT,
759 a,
760 b,
761 log2: l.log2_block,
762 format: l.format,
763 numbered: l.format == store_io_format::slot::FORMAT_2,
764 }
765}
766
767pub(crate) fn pair_table(volume: VolumeId, l: &Layout) -> PairAt {
768 let (a, b) = l.table_slots();
769 PairAt {
770 volume: *volume.as_bytes(),
771 object: TABLE_OBJECT,
772 a,
773 b,
774 log2: l.log2_table,
775 format: l.format,
776 numbered: false,
777 }
778}
779
780pub(crate) fn pair_region(volume: VolumeId, l: &Layout, e: &TableEntry) -> PairAt {
781 PairAt {
782 volume: *volume.as_bytes(),
783 object: l.region_object(u64::from(e.region_id)),
784 a: e.offset,
785 b: e.offset + l.block(),
786 log2: l.log2_block,
787 format: l.format,
788 numbered: false,
789 }
790}