Skip to main content

store_io_engine/
open.rs

1//! Opening a store: finding the volume record, reading the region table,
2//! verifying every region against its header, and assembling the handle.
3
4use std::path::Path;
5use std::sync::{Arc, Mutex};
6
7use store_io_buf::BufPool;
8use store_io_core::decide::decide;
9use store_io_core::error::{Error, ErrorContext, Named, NotWrittenCause, Op, OsError};
10use store_io_core::id::{Generation, RegionId, VolumeId};
11use store_io_core::observe::Observer;
12use store_io_format::meta::{
13    RegionKind, RegionMeta, RegionState, TableEntry, VolumeMeta, decode_table,
14};
15use store_io_format::pair::Winner;
16use store_io_platform::{FileMode, IoOp, Platform, Queue, QueueConfig};
17
18use crate::domain::Domain;
19use crate::exec::{Lane, QueueSet, Ran, run};
20use crate::frontier::AppendFrontier;
21use crate::gate::Gate;
22use crate::layout::Layout;
23use crate::slots::{PairRead, read_pair};
24use crate::space::Accounting;
25use crate::store::*;
26
27impl<P: Platform> Store<P>
28where
29    P::Queue: Send,
30{
31    /// Opens an existing store for reading and writing.
32    ///
33    /// Takes the ownership lock, makes a dead owner's completed writes
34    /// durable (one full flush), reads the metadata, verifies every ready
35    /// region, and bumps the owner generation. Modifies no caller bytes.
36    ///
37    /// # Errors
38    ///
39    /// [`Error::NotFound`], [`Error::Locked`] (another handle, in this or
40    /// another process, holds the store), refusal errors,
41    /// [`Error::Corruption`] for unreadable or foreign metadata,
42    /// [`Error::Unsupported`] (`FormatVersion`) for a newer format and for
43    /// a format-1 store (read-only in 0.5: open it with
44    /// [`Store::open_readonly`], or copy it to format 2 with
45    /// [`copy_store`](crate::copy_store)) before anything is written,
46    /// [`Error::DurabilityUnknown`] when a dead owner's writes may not be
47    /// durable: the drain's flush failed, or the probe did (on macOS its
48    /// flush check is a flush; elsewhere it reads, and fails only on an I/O
49    /// error),
50    /// [`Error::Io`] (`Io(Lock)` on a file system that cannot lock).
51    pub fn open(platform: P, path: &Path, opts: StoreOptions) -> Result<Self, Error> {
52        Self::open_inner(platform, path, opts, false)
53    }
54
55    /// Opens an existing store read-only: no write rights, nothing modified,
56    /// no refusal on unsafe or unverified devices (reading is always allowed).
57    ///
58    /// # Errors
59    ///
60    /// As [`Store::open`], without the refusal errors.
61    pub fn open_readonly(platform: P, path: &Path, opts: StoreOptions) -> Result<Self, Error> {
62        Self::open_inner(platform, path, opts, true)
63    }
64
65    fn open_inner(
66        platform: P,
67        path: &Path,
68        opts: StoreOptions,
69        read_only: bool,
70    ) -> Result<Self, Error> {
71        let dir = platform
72            .open_dir(path, false)
73            .map_err(|e| not_found_or(Op::Open, e, Named::Store))?;
74        let name = container_name()?;
75        // A read-only open needs no write rights (a read-only mount, a
76        // backup copy): one read-only handle and the shared lock, which
77        // other readers may hold too and which excludes the writer (D-092).
78        let mode = if read_only {
79            FileMode::ReadOnly
80        } else {
81            FileMode::ReadWrite
82        };
83        let file = platform
84            .open_file(&dir, &name, mode)
85            .map_err(|e| not_found_or(Op::Open, e, Named::Store))?;
86        let lock = if read_only {
87            platform.lock_shared(&file)
88        } else {
89            platform.lock_exclusive(&file)
90        }
91        .map_err(|raw| {
92            // Another owner holds it; anything else (a file system that
93            // cannot lock) is a plain failure, not "locked".
94            if store_io_core::errno::is_lock_held(raw) {
95                Error::Locked
96            } else {
97                io(Op::Lock, raw)
98            }
99        })?;
100        // A writable open's probe may flush (the macOS flush check): its
101        // failure leaves a dead owner's writes unknown, as the drain's below
102        // does. A reader's probe never flushes.
103        let evidence = platform.probe(&dir, &file).map_err(|e| {
104            if read_only {
105                io(Op::Probe, e)
106            } else {
107                Error::DurabilityUnknown {
108                    op: Op::Probe,
109                    raw: Some(e),
110                    ctx: ErrorContext::default(),
111                }
112            }
113        })?;
114        let decision = decide(&evidence, &opts.trust);
115        if !read_only {
116            refuse(&decision)?;
117            // Drain a dead owner's completed-but-unflushed writes before
118            // reading any slot (a later write to the other slot could
119            // otherwise lose both copies at the next power cut).
120            platform
121                .flush_all(&file)
122                .map_err(|e| Error::DurabilityUnknown {
123                    op: Op::FlushAll,
124                    raw: Some(e),
125                    ctx: ErrorContext::default(),
126                })?;
127        }
128        let policy = policy_of(&decision);
129        let lanes = if read_only || !policy.flush {
130            1
131        } else {
132            flush_slots(&platform, &opts)
133        };
134        let queues = make_queues(&platform, &opts, lanes)?;
135        let container_len = platform.size(&file).map_err(|e| io(Op::Open, e))?;
136        // Geometry is in the volume record; the record's own location depends
137        // on it, so try every legal block size.
138        let boot_pool = make_pool(&evidence, &Layout::v1(16, 16), &opts, false)?;
139        let (volume, layout, found, volume_pair) = {
140            let mut q = queues.get(0);
141            find_volume(&mut q, &file, &boot_pool)?
142        };
143        let pool = make_pool(&evidence, &layout, &opts, true)?;
144        let vmeta = match found {
145            // Format 1 is read-only from 0.5 (D-112, D-116): refused before
146            // any write (nothing above wrote).
147            Found::V1(_) if !read_only => {
148                return Err(Error::Unsupported {
149                    what: store_io_core::error::Capability::FormatVersion,
150                });
151            }
152            Found::V1(vm) => vm,
153            Found::V2(vm) => {
154                let opened = {
155                    let mut q = queues.get(0);
156                    crate::v2::open_metadata::<P>(
157                        &file,
158                        &mut q,
159                        &pool,
160                        &layout,
161                        volume,
162                        vm,
163                        volume_pair,
164                        container_len,
165                        read_only,
166                        &policy,
167                        opts.append_window,
168                        &mut || {
169                            let cfg = QueueConfig {
170                                depth: opts.queue_depth.max(1),
171                                parallel: true,
172                                pool_threads: opts.t3_pool_threads,
173                                ..QueueConfig::default()
174                            };
175                            platform.queue(cfg).ok().map(|q| (q, cfg.depth as usize))
176                        },
177                    )?
178                };
179                let report = StoreReport {
180                    evidence,
181                    decision,
182                    block_size: layout.block(),
183                    table_capacity: layout.pages as usize
184                        * usize::from(store_io_format::meta2::entries_per_page(layout.log2_table)),
185                    volume,
186                    owner_generation: opened.owner_generation,
187                    read_only: opened.read_only,
188                    io: platform.io_path(),
189                    unavailable: opened.unavailable,
190                    format: store_io_format::slot::FORMAT_2,
191                    metadata: opened.report,
192                    flushes_in_flight: 1,
193                };
194                return Ok(Self::assemble(
195                    platform,
196                    dir,
197                    file,
198                    lock,
199                    volume,
200                    layout,
201                    policy,
202                    pool,
203                    queues,
204                    opened.meta,
205                    opened.regions,
206                    report,
207                    opened.read_only,
208                    opened.owner_generation,
209                    opts,
210                ));
211            }
212        };
213        let tat = pair_table(volume, &layout);
214        let mut q = queues.get(0);
215        let table_pair = read_pair(&mut q, &file, &pool, &tat)?;
216        let table_bytes = winner_payload(&table_pair)?;
217        let table = decode_table(&table_bytes, layout.block()).map_err(|_| corrupt())?;
218        let next_region_id = table.next_region_id;
219        let entries: Vec<TableEntry> = table.iter().collect();
220        let mut regions = Vec::new();
221        // A region that fails any check below is reported, with the reason,
222        // and never used as ready (D-090).
223        let mut unavailable = Vec::new();
224        for e in entries.iter().filter(|e| e.state == RegionState::Ready) {
225            let mut skip = |reason| {
226                unavailable.push(UnavailableRegion {
227                    name: e.name.as_str().to_owned(),
228                    id: RegionId::from_raw(u64::from(e.region_id)),
229                    kind: e.kind,
230                    reason,
231                });
232            };
233            let end = layout
234                .extent(e.data_size)
235                .and_then(|l| e.offset.checked_add(l))
236                .ok_or_else(corrupt)?;
237            if end > container_len {
238                // The table names space the container does not hold.
239                skip(Unavailable::ContainerShort {
240                    needed: end,
241                    actual: container_len,
242                });
243                continue;
244            }
245            let at = pair_region(volume, &layout, e);
246            let header = read_pair(&mut q, &file, &pool, &at)?;
247            let (Some(bytes), Some((_, info))) =
248                (header.payload.clone(), header.report.winner_info())
249            else {
250                skip(Unavailable::HeaderUnreadable);
251                continue;
252            };
253            let Ok(rm) = RegionMeta::decode(&bytes) else {
254                skip(Unavailable::HeaderInvalid);
255                continue;
256            };
257            if rm.region_id != e.region_id || rm.kind != e.kind {
258                skip(Unavailable::HeaderMismatch);
259                continue;
260            }
261            if rm.state != RegionState::Ready {
262                skip(Unavailable::HeaderReleased);
263                continue;
264            }
265            let Some(generation) = Generation::from_raw(info.generation) else {
266                skip(Unavailable::HeaderInvalid);
267                continue;
268            };
269            let frontier = (e.kind == RegionKind::Append).then(|| {
270                AppendFrontier::new(
271                    0,
272                    e.data_size,
273                    u32::from(layout.log2_block),
274                    crate::frontier::ring_for(
275                        opts.append_window,
276                        e.data_size,
277                        u32::from(layout.log2_block),
278                    ),
279                )
280            });
281            regions.push(Arc::new(LiveRegion {
282                id: RegionId::from_raw(u64::from(e.region_id)),
283                name: e.name,
284                kind: e.kind,
285                generation,
286                map: crate::regionmap::RegionMap::single(
287                    e.offset + 2 * layout.block(),
288                    e.data_size,
289                ),
290                header_at: [e.offset, e.offset + layout.block()],
291                header: Mutex::new(header),
292                frontier,
293                positioned: std::sync::atomic::AtomicBool::new(false),
294                force_sync: Arc::new(std::sync::atomic::AtomicBool::new(false)),
295                read_only: std::sync::atomic::AtomicBool::new(false),
296                superseded: false,
297                foreign: false,
298                gate: Gate::new(),
299            }));
300        }
301        drop(q);
302        let meta = Meta {
303            volume: vmeta,
304            volume_pair,
305            table_pair,
306            next_region_id,
307            entries,
308            container_len,
309            v2: None,
310            unavailable: Vec::new(),
311            repairs: Vec::new(),
312        };
313        // Format 1 opens for reading only (a writable open was refused
314        // above): no owner generation is taken, nothing is written.
315        debug_assert!(read_only, "a writable format-1 open was refused");
316        let owner_generation = 0;
317        let report = StoreReport {
318            evidence,
319            decision,
320            block_size: layout.block(),
321            table_capacity: layout.table_capacity(),
322            volume,
323            owner_generation,
324            read_only,
325            io: platform.io_path(),
326            unavailable,
327            format: store_io_format::slot::FORMAT_1,
328            metadata: MetadataReport::default(),
329            flushes_in_flight: 1,
330        };
331        Ok(Self::assemble(
332            platform,
333            dir,
334            file,
335            lock,
336            volume,
337            layout,
338            policy,
339            pool,
340            queues,
341            meta,
342            regions,
343            report,
344            read_only,
345            owner_generation,
346            opts,
347        ))
348    }
349
350    #[allow(clippy::too_many_arguments)]
351    pub(crate) fn assemble(
352        platform: P,
353        dir: P::Dir,
354        file: P::File,
355        lock: P::Lock,
356        volume: VolumeId,
357        layout: Layout,
358        policy: Policy,
359        pool: BufPool,
360        mut queues: QueueSet<P::Queue>,
361        meta: Meta,
362        regions: Vec<Arc<LiveRegion>>,
363        report: StoreReport,
364        read_only: bool,
365        owner_generation: u64,
366        opts: StoreOptions,
367    ) -> Self {
368        let acct = match meta.v2.as_deref() {
369            Some(v2) => Accounting::from_usage(&v2.usage),
370            None => Accounting::from_entries(&meta.entries, layout.block()),
371        };
372        // Every flush lane past the first needs a handle of its own. A store
373        // that flushes nothing (read-only: asked for, or found so by a
374        // format-2 open after its queues were made; or on a device that
375        // needs no flush) opens none, and a lane whose handle could not be
376        // made is closed.
377        let lanes = if read_only || !policy.flush {
378            1
379        } else {
380            queues.flush_lanes()
381        };
382        let flush_files = flush_handles(&platform, &file, lanes);
383        queues.keep_flush_lanes(1 + flush_files.len());
384        let mut report = report;
385        report.flushes_in_flight = 1 + flush_files.len() as u8;
386        let mut meta = meta;
387        meta.unavailable = report.unavailable.clone();
388        let frozen = std::sync::atomic::AtomicBool::new(report.metadata.frozen);
389        let block = layout.block() as usize;
390        let ordinary = crate::store::io_max(&opts).max(pool.min_len());
391        let chunk = (ordinary / block * block).max(block);
392        Self {
393            inner: Arc::new(Inner {
394                domain: Domain::with_slots(1 + flush_files.len()),
395                flush_files: flush_files.into_boxed_slice(),
396                platform,
397                _dir: dir,
398                file,
399                _lock: lock,
400                volume,
401                layout,
402                policy,
403                pool,
404                queues,
405                meta: Mutex::new(meta),
406                regions: Mutex::new(regions),
407                report,
408                frozen,
409                read_only,
410                owner_generation,
411                opts,
412                ordinary,
413                chunk,
414                acct: Mutex::new(acct),
415                epoch: EPOCH.fetch_add(1, std::sync::atomic::Ordering::Relaxed),
416                observer: Observer::new(),
417                seen: Mutex::new(crate::stats::Seen::default()),
418                submitters: Mutex::new(crate::stats::Submitters::default()),
419                flusher: std::sync::OnceLock::new(),
420            }),
421        }
422    }
423}
424
425/// Handles for flush slots 1.. (D-124): the container itself reopened
426/// (`Platform::reopen`), once per slot past the first, each its own open
427/// file description (so each flush reports its own failures; the ownership
428/// lock stays on slot 0's, and closing these never releases it). Best
429/// effort: a handle that cannot be made leaves fewer slots, which the report
430/// states.
431fn flush_handles<P: Platform>(platform: &P, file: &P::File, lanes: usize) -> Vec<P::File> {
432    (1..lanes).map_while(|_| platform.reopen(file)).collect()
433}
434
435/// The volume record found, by format.
436enum Found {
437    V1(VolumeMeta),
438    V2(store_io_format::meta2::VolumeMeta2),
439}
440
441/// Finds the volume record by trying every legal block size; the slot's
442/// `format_major` says which format to read it as.
443fn find_volume<Q: Queue>(
444    q: &mut Lane<Q>,
445    file: &Q::File,
446    pool: &BufPool,
447) -> Result<(VolumeId, Layout, Found, PairRead), Error> {
448    // Read the first slot header at the smallest size to learn the volume id;
449    // either copy may be damaged, so try both locations per size.
450    let mut media_error = false;
451    for log2 in crate::layout::MIN_LOG2_BLOCK..=crate::layout::MAX_LOG2_BLOCK {
452        let size = 1u64 << log2;
453        for probe_at in [0, size] {
454            // The boot pool, which has no large buffers.
455            let buf = pool.take(4096).map_err(|_| Error::NotWritten {
456                cause: NotWrittenCause::PoolExhausted,
457                ctx: ErrorContext::default(),
458            })?;
459            // Only an unreadable location (a media error, a short read, the
460            // end of the file) moves on to the next candidate. Any other
461            // failure is returned: treating it as "no volume here" would
462            // report a healthy store as corrupt (found by review).
463            let io = |raw| Error::Io {
464                op: Op::Read,
465                raw,
466                ctx: ErrorContext::default(),
467            };
468            let (got, result) = match run(
469                q,
470                IoOp::Read {
471                    file,
472                    offset: probe_at,
473                    buf,
474                },
475            ) {
476                Ok(Ran::Done { buf, result }) => (buf, result),
477                // A failed wait says nothing about this candidate: stop.
478                Ok(Ran::Abandoned(raw)) | Err((_, Some(raw))) => return Err(io(raw)),
479                Err((_, None)) => {
480                    return Err(Error::NotWritten {
481                        cause: NotWrittenCause::QueueFull,
482                        ctx: ErrorContext::default(),
483                    });
484                }
485            };
486            match result {
487                Err(raw) if !crate::slots::unreadable(raw) => return Err(io(raw)),
488                Ok(4096) => {}
489                Err(raw) => {
490                    media_error |= store_io_core::errno::is_media_error(raw);
491                    continue;
492                }
493                Ok(_) => continue,
494            }
495            let Some(b) = got else { continue };
496            if b.as_slice()[..8] != store_io_format::slot::MAGIC {
497                continue;
498            }
499            let mut uuid = [0u8; 16];
500            uuid.copy_from_slice(&b.as_slice()[32..48]);
501            if b.as_slice()[78] != log2 {
502                continue;
503            }
504            // The format comes only from a header whose CRC verifies (spec
505            // ยง2): a bit flip there must not decide how the pair is read.
506            if !header_verifies(b.as_slice()) {
507                continue;
508            }
509            let format = u16::from_le_bytes([b.as_slice()[14], b.as_slice()[15]]);
510            let v2 = format == store_io_format::slot::FORMAT_2;
511            let layout0 = Layout {
512                format: if v2 {
513                    format
514                } else {
515                    store_io_format::slot::FORMAT_1
516                },
517                ..Layout::v1(log2, log2)
518            };
519            let at = pair_volume(VolumeId::from_bytes(uuid), &layout0);
520            let pair = read_pair(q, file, pool, &at)?;
521            media_error |= pair.media_error;
522            let bytes = match winner_payload(&pair) {
523                Ok(b) => b,
524                Err(Error::Unsupported { what }) => return Err(Error::Unsupported { what }),
525                Err(_) => continue,
526            };
527            // A longer record, or a feature this version does not know, is a
528            // newer format: refused as unsupported (and never rewritten),
529            // not reported as corruption.
530            let newer = Error::Unsupported {
531                what: store_io_core::error::Capability::FormatVersion,
532            };
533            if v2 {
534                let vm = crate::v2::decode_volume(&bytes)?;
535                if vm.base.log2_block_size != log2 {
536                    continue;
537                }
538                let layout = Layout {
539                    log2_block: log2,
540                    log2_table: vm.base.log2_table_slot_size,
541                    format,
542                    pages: vm.table_pages,
543                };
544                return Ok((VolumeId::from_bytes(uuid), layout, Found::V2(vm), pair));
545            }
546            if bytes.len() > store_io_format::meta::VOLUME_META_LEN {
547                return Err(newer);
548            }
549            let vm = VolumeMeta::decode(&bytes).map_err(|e| match e {
550                store_io_format::meta::MetaError::Field("features") => newer,
551                _ => corrupt(),
552            })?;
553            if vm.log2_block_size != log2 {
554                continue;
555            }
556            let layout = Layout::v1(log2, vm.log2_table_slot_size);
557            if vm.table_offset != layout.table_slots().0 {
558                return Err(corrupt());
559            }
560            return Ok((VolumeId::from_bytes(uuid), layout, Found::V1(vm), pair));
561        }
562    }
563    // No volume record found. If a read failed on the media on the way (a
564    // probe or a slot of a candidate pair), that is the answer: it may be
565    // transient, and "corrupt" would invite a caller to recreate the store
566    // over its data. A media error at a candidate that was never the record
567    // also ends here; erring toward "do not recreate" is deliberate.
568    Err(Error::Corruption {
569        kind: if media_error {
570            store_io_core::error::CorruptionKind::MediaError
571        } else {
572            store_io_core::error::CorruptionKind::HeaderCrc
573        },
574        ctx: ErrorContext::default(),
575    })
576}
577
578/// Source of [`Inner::epoch`].
579static EPOCH: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1);
580
581fn not_found_or(op: Op, raw: OsError, what: Named) -> Error {
582    if store_io_core::errno::is_not_found(raw) {
583        Error::NotFound { what }
584    } else {
585        io(op, raw)
586    }
587}
588
589pub(crate) fn winner_payload(p: &PairRead) -> Result<Vec<u8>, Error> {
590    match p.report.winner {
591        Winner::Confirmed(_) | Winner::Unconfirmed(_) => p.payload.clone().ok_or_else(corrupt),
592        Winner::Refused => Err(Error::Unsupported {
593            what: store_io_core::error::Capability::FormatVersion,
594        }),
595        Winner::Fork => Err(Error::Corruption {
596            kind: store_io_core::error::CorruptionKind::Fork,
597            ctx: ErrorContext::default(),
598        }),
599        // No version left, and a slot was lost to a media error: report that
600        // (it may be transient) rather than corruption, which would invite a
601        // caller to recreate the store over its data.
602        Winner::Lost if p.media_error => Err(Error::Corruption {
603            kind: store_io_core::error::CorruptionKind::MediaError,
604            ctx: ErrorContext::default(),
605        }),
606        Winner::Empty | Winner::Lost => Err(Error::Corruption {
607            kind: store_io_core::error::CorruptionKind::HeaderCrc,
608            ctx: ErrorContext::default(),
609        }),
610    }
611}
612
613/// Whether the slot header at the start of `b` carries a header CRC that
614/// verifies. A header longer than `b` (format 1 allows one; format 2's is
615/// exactly 128 bytes) is not judged here: true, unless it claims format 2.
616fn header_verifies(b: &[u8]) -> bool {
617    let len = usize::from(u16::from_le_bytes([b[12], b[13]]));
618    if len > b.len() {
619        return u16::from_le_bytes([b[14], b[15]]) != store_io_format::slot::FORMAT_2;
620    }
621    let Some(h) = b.get(..len.max(store_io_format::slot::HEADER_LEN)) else {
622        return false;
623    };
624    let stored = u32::from_le_bytes([h[8], h[9], h[10], h[11]]);
625    let mut h = h.to_vec();
626    h[8..12].fill(0);
627    store_io_format::crc32c::crc32c(&h) == stored
628}