Skip to main content

store_io/
native.rs

1//! The simple API on the operating system's native backend.
2
3use std::ops::Deref;
4use std::path::Path;
5
6use store_io_core::decide::{Trust, decide};
7use store_io_core::error::{Error, ErrorContext, Named, NotWrittenCause, Op, OsError};
8use store_io_platform::{FileName, LinuxTier, Platform};
9
10use crate::{
11    CopyOptions, CopyReport, FileSinkOptions, FileSourceOptions, ImportError, ImportOptions,
12    ImportReport, ImportSource, NativePlatform, Probe, StoreOptions, StoreSinkOptions,
13};
14
15/// The native platform on the kernel interface `tier` asks for (D-082;
16/// only Linux has a choice).
17fn platform(tier: LinuxTier) -> Result<NativePlatform, Error> {
18    #[cfg(target_os = "linux")]
19    {
20        NativePlatform::with_tier(tier).map_err(|raw| io(Op::Queue, raw))
21    }
22    #[cfg(not(target_os = "linux"))]
23    {
24        let _ = tier;
25        Ok(NativePlatform::new())
26    }
27}
28
29/// Paths accepted by the simple API.
30fn path_of<P: AsRef<Path> + ?Sized>(p: &P) -> &Path {
31    p.as_ref()
32}
33
34/// A region store-io appends to (see [`store_io_engine::AppendRegion`]).
35pub type AppendRegion = store_io_engine::AppendRegion<NativePlatform>;
36/// A region of caller-addressed blocks (see [`store_io_engine::PageRegion`]).
37pub type PageRegion = store_io_engine::PageRegion<NativePlatform>;
38/// A small object committed atomically (see [`store_io_engine::Slot`]).
39pub type Slot = store_io_engine::Slot<NativePlatform>;
40/// Many appends, one barrier (see [`store_io_engine::AppendBatch`]).
41pub type AppendBatch = store_io_engine::AppendBatch<NativePlatform>;
42/// Many page writes, one barrier (see [`store_io_engine::PageBatch`]).
43pub type PageBatch = store_io_engine::PageBatch<NativePlatform>;
44/// Space set aside for provisioning (see [`store_io_engine::Reservation`]).
45pub type Reservation = store_io_engine::Reservation<NativePlatform>;
46/// Many reads and writes in flight from one thread, without blocking (see
47/// [`store_io_engine::Submitter`]); from [`Store::submitter`](store_io_engine::Store::submitter).
48pub type Submitter = store_io_engine::Submitter<NativePlatform>;
49/// Stores on several devices, made durable together (see
50/// [`store_io_engine::StoreSet`]); built from [`Store`]s.
51pub type StoreSet = store_io_engine::StoreSet<NativePlatform>;
52
53/// A region's bytes in order as owned chunks (`read_stream` on a region).
54pub type ReadStream = store_io_engine::ReadStream<NativePlatform>;
55
56/// One range for `Store::prefetch` (`stream_range` on a region).
57pub type StreamRange<'a> = store_io_engine::StreamRange<'a, NativePlatform>;
58
59/// An export written to a directory ([`file_sink`]).
60pub type FileSink = store_io_engine::FileSink<NativePlatform>;
61/// An export read back from a directory ([`file_source`]).
62pub type FileSource = store_io_engine::FileSource<NativePlatform>;
63/// An export restored straight into a new store ([`store_sink`]).
64pub type StoreSink = store_io_engine::StoreSink<NativePlatform>;
65
66/// A store: one container file in a directory of its own, on one device.
67///
68/// Every method of [`store_io_engine::Store`] is available through `Deref`:
69/// provisioning and looking up regions and slots, `reserve`, `space`,
70/// `report`, `poison`.
71///
72/// ```no_run
73/// use store_io::{Store, StoreOptions};
74///
75/// let store = Store::create("data/orders", StoreOptions::default())?;
76/// println!("{}", store.report());
77/// drop(store);
78/// let store = Store::open("data/orders")?;
79/// let wal = store.append_region("wal");
80/// # let _ = wal;
81/// # Ok::<(), store_io::Error>(())
82/// ```
83#[derive(Clone, Debug)]
84pub struct Store(store_io_engine::Store<NativePlatform>);
85
86impl Store {
87    /// Creates a new store in `path` (the directory is created if needed).
88    /// The device is probed first: unsafe or unverified devices are refused
89    /// unless `opts.trust` attests or overrides (and then every receipt is
90    /// labelled accordingly).
91    ///
92    /// # Errors
93    ///
94    /// `AlreadyExists`, `UnsafeDevice`, `Unverified`, `NoSpace`, `Io`.
95    pub fn create(path: impl AsRef<Path>, opts: StoreOptions) -> Result<Self, Error> {
96        store_io_engine::Store::create(platform(opts.linux_tier)?, path_of(&path), opts).map(Self)
97    }
98
99    /// Opens an existing store for writing, with default options. Takes the
100    /// ownership lock, waits out a dead owner's writes, and verifies every
101    /// region against its header. Append regions must be positioned with
102    /// `resume_at` before the first append.
103    ///
104    /// # Errors
105    ///
106    /// `NotFound`, `Locked`, `UnsafeDevice`, `Unverified`, `Corruption`,
107    /// `DurabilityUnknown` (a dead owner's writes could not be made
108    /// durable), `Io`; `Unsupported(FormatVersion)` for a newer format and for a
109    /// store of container format 1 (before 0.5), which opens read-only
110    /// ([`Self::open_readonly`]) and moves to format 2 by [`copy_store`]
111    /// (`store-io copy`).
112    pub fn open(path: impl AsRef<Path>) -> Result<Self, Error> {
113        Self::open_with(path, StoreOptions::default())
114    }
115
116    /// Opens an existing store for writing with explicit options.
117    ///
118    /// # Errors
119    ///
120    /// As [`Self::open`].
121    pub fn open_with(path: impl AsRef<Path>, opts: StoreOptions) -> Result<Self, Error> {
122        store_io_engine::Store::open(platform(opts.linux_tier)?, path_of(&path), opts).map(Self)
123    }
124
125    /// Opens an existing store for reading and recovery only: no write
126    /// rights, and nothing on disk changes.
127    ///
128    /// # Errors
129    ///
130    /// `NotFound`, `Corruption`, `Io`.
131    pub fn open_readonly(path: impl AsRef<Path>) -> Result<Self, Error> {
132        Self::open_readonly_with(path, StoreOptions::default())
133    }
134
135    /// Opens an existing store for reading and recovery only, with
136    /// explicit options (for example `linux_tier: LinuxTier::Sync`).
137    ///
138    /// # Errors
139    ///
140    /// As [`Self::open_readonly`].
141    pub fn open_readonly_with(path: impl AsRef<Path>, opts: StoreOptions) -> Result<Self, Error> {
142        store_io_engine::Store::open_readonly(platform(opts.linux_tier)?, path_of(&path), opts)
143            .map(Self)
144    }
145
146    /// The engine store underneath.
147    #[must_use]
148    pub fn engine(&self) -> &store_io_engine::Store<NativePlatform> {
149        &self.0
150    }
151}
152
153impl Deref for Store {
154    type Target = store_io_engine::Store<NativePlatform>;
155
156    fn deref(&self) -> &Self::Target {
157        &self.0
158    }
159}
160
161impl From<store_io_engine::Store<NativePlatform>> for Store {
162    fn from(s: store_io_engine::Store<NativePlatform>) -> Self {
163        Self(s)
164    }
165}
166
167impl From<Store> for store_io_engine::Store<NativePlatform> {
168    fn from(s: Store) -> Self {
169        s.0
170    }
171}
172
173/// A directory of small files, each replaced atomically and durably (see
174/// [`store_io_engine::Directory`]).
175///
176/// ```no_run
177/// use store_io::Directory;
178///
179/// let dir = Directory::open("data/meta", true)?;
180/// dir.replace("CURRENT", b"MANIFEST-000042")?;
181/// assert_eq!(dir.read("CURRENT")?, b"MANIFEST-000042");
182/// # Ok::<(), store_io::Error>(())
183/// ```
184#[derive(Debug)]
185pub struct Directory(store_io_engine::Directory<NativePlatform>);
186
187impl Directory {
188    /// Opens `path` (creating it if `create`), with default trust.
189    ///
190    /// # Errors
191    ///
192    /// `NotFound`, `Io`.
193    pub fn open(path: impl AsRef<Path>, create: bool) -> Result<Self, Error> {
194        Self::open_with(path, create, Trust::default())
195    }
196
197    /// Opens `path` with explicit trust inputs.
198    ///
199    /// # Errors
200    ///
201    /// As [`Self::open`].
202    pub fn open_with(path: impl AsRef<Path>, create: bool, trust: Trust) -> Result<Self, Error> {
203        store_io_engine::Directory::open(platform(LinuxTier::Auto)?, path_of(&path), create, trust)
204            .map(Self)
205    }
206
207    /// The largest object a handle writes or reads by default (see
208    /// [`Self::with_max_object`]): 16 MiB.
209    pub const DEFAULT_MAX_OBJECT: u64 =
210        store_io_engine::Directory::<NativePlatform>::DEFAULT_MAX_OBJECT;
211
212    /// The handle with `max` as its largest object, written or read (see
213    /// [`store_io_engine::Directory::with_max_object`]): a larger `replace`
214    /// is refused before anything is written, and a larger file by `read`
215    /// before anything is allocated.
216    #[must_use]
217    pub fn with_max_object(self, max: u64) -> Self {
218        Self(self.0.with_max_object(max))
219    }
220}
221
222impl Deref for Directory {
223    type Target = store_io_engine::Directory<NativePlatform>;
224
225    fn deref(&self) -> &Self::Target {
226        &self.0
227    }
228}
229
230fn io(op: Op, raw: OsError) -> Error {
231    Error::Io {
232        op,
233        raw,
234        ctx: ErrorContext::default(),
235    }
236}
237
238/// Copies the store in `from` into a new store (container format 2) in
239/// `to`, whose directory must hold no store: the way a format-1 store
240/// moves to format 2, and the way to compact a format-2 one. The source is
241/// opened read-only and never written; the copy is built under a temporary
242/// name and renamed into place only when whole and durable, so after a
243/// crash `to` holds nothing or the whole copy (run the copy again).
244///
245/// ```no_run
246/// use store_io::CopyOptions;
247///
248/// let r = store_io::copy_store("data/orders", "data/orders-2", CopyOptions::default())?;
249/// println!("{} regions, {} bytes", r.regions, r.bytes);
250/// # Ok::<(), store_io::Error>(())
251/// ```
252///
253/// # Errors
254///
255/// As [`store_io_engine::copy_store`], among them: `AlreadyExists` if `to`
256/// holds a store; `Locked` while a writer holds `from` open or another
257/// copy into `to` runs;
258/// `NotWritten(NotReady | Frozen | TooLarge)` for a source it cannot copy
259/// whole, before anything is created; `NotWritten(Misaligned)` if `to`'s
260/// device needs larger blocks than the source's; `NotFound`, `Corruption`
261/// or `Unsupported` opening the source; `UnsafeDevice`, `Unverified` (the
262/// target's device refused: see `StoreOptions::trust`), `NoSpace`, `Io`;
263/// `DurabilityUnknown` if the final rename or directory flush failed (the
264/// copy may or may not be in place).
265pub fn copy_store(
266    from: impl AsRef<Path>,
267    to: impl AsRef<Path>,
268    opts: CopyOptions,
269) -> Result<CopyReport, Error> {
270    store_io_engine::copy_store(
271        platform(opts.store.linux_tier)?,
272        path_of(&from),
273        path_of(&to),
274        opts,
275    )
276}
277
278/// A sink writing an export into the existing, empty directory `dir`: one
279/// data file per region and a `regions.info` record, restorable only once
280/// the export returned `Ok`.
281///
282/// ```no_run
283/// let store = store_io::Store::open("data/orders")?;
284/// let mut sink = store_io::file_sink("backup/orders", store_io::FileSinkOptions::default())?;
285/// let report = store.export(&mut sink).map_err(|e| e.error)?;
286/// assert!(report.complete);
287/// # Ok::<(), store_io::Error>(())
288/// ```
289///
290/// # Errors
291///
292/// As [`store_io_engine::FileSink::create`]: `NotFound` if `dir` does not
293/// exist, `NotWritten(InvalidOptions)` if it is not empty, `Io`.
294pub fn file_sink(dir: impl AsRef<Path>, opts: FileSinkOptions) -> Result<FileSink, Error> {
295    store_io_engine::FileSink::create(platform(opts.store.linux_tier)?, path_of(&dir), opts)
296}
297
298/// A source reading back the export a [`file_sink`] wrote into `dir`, for
299/// [`import`]: every span of every data file checked against its CRC-32C
300/// before its bytes are handed out.
301///
302/// # Errors
303///
304/// As [`store_io_engine::FileSource::open`]: `NotFound` if `dir` does not
305/// exist, `Corruption` if it holds no `regions.info`, `Io`.
306pub fn file_source(dir: impl AsRef<Path>, opts: FileSourceOptions) -> Result<FileSource, Error> {
307    store_io_engine::FileSource::open(platform(opts.store.linux_tier)?, path_of(&dir), opts)
308}
309
310/// Imports an export into a new store in `to`, whose directory must hold
311/// no store: built under a temporary name and renamed into place only when
312/// whole and durable, so after a crash `to` holds nothing or the whole
313/// store (import again).
314///
315/// ```no_run
316/// let mut source = store_io::file_source("backup/orders", Default::default())?;
317/// let r = store_io::import(&mut source, "data/orders", store_io::ImportOptions::default())
318///     .map_err(|e| e.error)?;
319/// println!("{} regions, {} bytes", r.regions.len(), r.bytes);
320/// # Ok::<(), store_io::Error>(())
321/// ```
322///
323/// # Errors
324///
325/// As [`store_io_engine::import`] (boxed, with how far it got).
326pub fn import(
327    source: &mut dyn ImportSource,
328    to: impl AsRef<Path>,
329    opts: ImportOptions,
330) -> Result<ImportReport, Box<ImportError>> {
331    let p = platform(opts.store.linux_tier)
332        .map_err(|error| Box::new(ImportError::from_error(error)))?;
333    store_io_engine::import(p, path_of(&to), source, opts)
334}
335
336/// A sink restoring an export straight into a new store in `to`, whose
337/// directory must hold no store (its copy lock held until the sink drops):
338/// each region rebuilt as [`import`] rebuilds it, the store renamed into
339/// place only once the export returned `Ok`, so after a crash `to` holds
340/// nothing or the whole store.
341///
342/// ```no_run
343/// let store = store_io::Store::open("data/orders")?;
344/// let mut sink = store_io::store_sink("replica/orders", store_io::StoreSinkOptions::default())?;
345/// let report = store.export(&mut sink).map_err(|e| e.error)?;
346/// assert!(report.complete && sink.is_complete());
347/// # Ok::<(), store_io::Error>(())
348/// ```
349///
350/// # Errors
351///
352/// As [`store_io_engine::StoreSink::create`]: `AlreadyExists` if a store
353/// is in `to`, `Locked` while another copy, import or sink into it runs,
354/// `Io`.
355pub fn store_sink(to: impl AsRef<Path>, opts: StoreSinkOptions) -> Result<StoreSink, Error> {
356    store_io_engine::StoreSink::create(platform(opts.store.linux_tier)?, path_of(&to), opts)
357}
358
359/// Probes the device under the existing directory `dir`: the evidence
360/// store-io reads and the durability class it would decide with default
361/// trust. Creates, probes and removes a small temporary file there.
362///
363/// ```no_run
364/// let p = store_io::probe("data")?;
365/// println!("{:?}: {:?}", p.decision.class, p.decision.reasons);
366/// # Ok::<(), store_io::Error>(())
367/// ```
368///
369/// # Errors
370///
371/// `NotFound`, `Io`.
372pub fn probe(dir: impl AsRef<Path>) -> Result<Probe, Error> {
373    let p = NativePlatform::new();
374    let d = p.open_dir(path_of(&dir), false).map_err(|raw| {
375        if store_io_core::errno::is_not_found(raw) {
376            Error::NotFound { what: Named::Store }
377        } else {
378            io(Op::Open, raw)
379        }
380    })?;
381    // Unique per call (process, a counter and a random part, for processes
382    // in different PID namespaces sharing a directory), so concurrent
383    // probes of one directory never share, or remove, each other's file.
384    static PROBES: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
385    let n = PROBES.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
386    let salt = {
387        use std::hash::{BuildHasher, Hasher};
388        let mut h = std::collections::hash_map::RandomState::new().build_hasher();
389        h.write_u64(n);
390        h.finish()
391    };
392    let unique = format!(".store-io-probe-{}-{n}-{salt:016x}", std::process::id());
393    let name = FileName::new(&unique).map_err(|_| Error::NotWritten {
394        cause: NotWrittenCause::InvalidName,
395        ctx: ErrorContext::default(),
396    })?;
397    let file = p
398        .create_file(&d, &name)
399        .map_err(|raw| io(Op::Create, raw))?;
400    let evidence = p
401        .allocate(&file, 4096)
402        .and_then(|()| p.probe(&d, &file))
403        .map_err(|raw| io(Op::Probe, raw));
404    drop(file);
405    let _removed = p.unlink(&d, &name);
406    let evidence = evidence?;
407    let decision = decide(&evidence, &Trust::default());
408    Ok(Probe { evidence, decision })
409}