store_io/native.rs
1//! The simple API on the operating system's native backend.
2
3use std::ops::Deref;
4use std::path::Path;
5
6use store_io_core::decide::{Trust, decide};
7use store_io_core::error::{Error, ErrorContext, Named, NotWrittenCause, Op, OsError};
8use store_io_platform::{FileName, LinuxTier, Platform};
9
10use crate::{
11 CopyOptions, CopyReport, FileSinkOptions, FileSourceOptions, ImportError, ImportOptions,
12 ImportReport, ImportSource, NativePlatform, Probe, StoreOptions, StoreSinkOptions,
13};
14
15/// The native platform on the kernel interface `tier` asks for (D-082;
16/// only Linux has a choice).
17fn platform(tier: LinuxTier) -> Result<NativePlatform, Error> {
18 #[cfg(target_os = "linux")]
19 {
20 NativePlatform::with_tier(tier).map_err(|raw| io(Op::Queue, raw))
21 }
22 #[cfg(not(target_os = "linux"))]
23 {
24 let _ = tier;
25 Ok(NativePlatform::new())
26 }
27}
28
29/// Paths accepted by the simple API.
30fn path_of<P: AsRef<Path> + ?Sized>(p: &P) -> &Path {
31 p.as_ref()
32}
33
34/// A region store-io appends to (see [`store_io_engine::AppendRegion`]).
35pub type AppendRegion = store_io_engine::AppendRegion<NativePlatform>;
36/// A region of caller-addressed blocks (see [`store_io_engine::PageRegion`]).
37pub type PageRegion = store_io_engine::PageRegion<NativePlatform>;
38/// A small object committed atomically (see [`store_io_engine::Slot`]).
39pub type Slot = store_io_engine::Slot<NativePlatform>;
40/// Many appends, one barrier (see [`store_io_engine::AppendBatch`]).
41pub type AppendBatch = store_io_engine::AppendBatch<NativePlatform>;
42/// Many page writes, one barrier (see [`store_io_engine::PageBatch`]).
43pub type PageBatch = store_io_engine::PageBatch<NativePlatform>;
44/// Space set aside for provisioning (see [`store_io_engine::Reservation`]).
45pub type Reservation = store_io_engine::Reservation<NativePlatform>;
46/// Many reads and writes in flight from one thread, without blocking (see
47/// [`store_io_engine::Submitter`]); from [`Store::submitter`](store_io_engine::Store::submitter).
48pub type Submitter = store_io_engine::Submitter<NativePlatform>;
49/// Stores on several devices, made durable together (see
50/// [`store_io_engine::StoreSet`]); built from [`Store`]s.
51pub type StoreSet = store_io_engine::StoreSet<NativePlatform>;
52
53/// A region's bytes in order as owned chunks (`read_stream` on a region).
54pub type ReadStream = store_io_engine::ReadStream<NativePlatform>;
55
56/// One range for `Store::prefetch` (`stream_range` on a region).
57pub type StreamRange<'a> = store_io_engine::StreamRange<'a, NativePlatform>;
58
59/// An export written to a directory ([`file_sink`]).
60pub type FileSink = store_io_engine::FileSink<NativePlatform>;
61/// An export read back from a directory ([`file_source`]).
62pub type FileSource = store_io_engine::FileSource<NativePlatform>;
63/// An export restored straight into a new store ([`store_sink`]).
64pub type StoreSink = store_io_engine::StoreSink<NativePlatform>;
65
66/// A store: one container file in a directory of its own, on one device.
67///
68/// Every method of [`store_io_engine::Store`] is available through `Deref`:
69/// provisioning and looking up regions and slots, `reserve`, `space`,
70/// `report`, `poison`.
71///
72/// ```no_run
73/// use store_io::{Store, StoreOptions};
74///
75/// let store = Store::create("data/orders", StoreOptions::default())?;
76/// println!("{}", store.report());
77/// drop(store);
78/// let store = Store::open("data/orders")?;
79/// let wal = store.append_region("wal");
80/// # let _ = wal;
81/// # Ok::<(), store_io::Error>(())
82/// ```
83#[derive(Clone, Debug)]
84pub struct Store(store_io_engine::Store<NativePlatform>);
85
86impl Store {
87 /// Creates a new store in `path` (the directory is created if needed).
88 /// The device is probed first: unsafe or unverified devices are refused
89 /// unless `opts.trust` attests or overrides (and then every receipt is
90 /// labelled accordingly).
91 ///
92 /// # Errors
93 ///
94 /// `AlreadyExists`, `UnsafeDevice`, `Unverified`, `NoSpace`, `Io`.
95 pub fn create(path: impl AsRef<Path>, opts: StoreOptions) -> Result<Self, Error> {
96 store_io_engine::Store::create(platform(opts.linux_tier)?, path_of(&path), opts).map(Self)
97 }
98
99 /// Opens an existing store for writing, with default options. Takes the
100 /// ownership lock, waits out a dead owner's writes, and verifies every
101 /// region against its header. Append regions must be positioned with
102 /// `resume_at` before the first append.
103 ///
104 /// # Errors
105 ///
106 /// `NotFound`, `Locked`, `UnsafeDevice`, `Unverified`, `Corruption`,
107 /// `DurabilityUnknown` (a dead owner's writes could not be made
108 /// durable), `Io`; `Unsupported(FormatVersion)` for a newer format and for a
109 /// store of container format 1 (before 0.5), which opens read-only
110 /// ([`Self::open_readonly`]) and moves to format 2 by [`copy_store`]
111 /// (`store-io copy`).
112 pub fn open(path: impl AsRef<Path>) -> Result<Self, Error> {
113 Self::open_with(path, StoreOptions::default())
114 }
115
116 /// Opens an existing store for writing with explicit options.
117 ///
118 /// # Errors
119 ///
120 /// As [`Self::open`].
121 pub fn open_with(path: impl AsRef<Path>, opts: StoreOptions) -> Result<Self, Error> {
122 store_io_engine::Store::open(platform(opts.linux_tier)?, path_of(&path), opts).map(Self)
123 }
124
125 /// Opens an existing store for reading and recovery only: no write
126 /// rights, and nothing on disk changes.
127 ///
128 /// # Errors
129 ///
130 /// `NotFound`, `Corruption`, `Io`.
131 pub fn open_readonly(path: impl AsRef<Path>) -> Result<Self, Error> {
132 Self::open_readonly_with(path, StoreOptions::default())
133 }
134
135 /// Opens an existing store for reading and recovery only, with
136 /// explicit options (for example `linux_tier: LinuxTier::Sync`).
137 ///
138 /// # Errors
139 ///
140 /// As [`Self::open_readonly`].
141 pub fn open_readonly_with(path: impl AsRef<Path>, opts: StoreOptions) -> Result<Self, Error> {
142 store_io_engine::Store::open_readonly(platform(opts.linux_tier)?, path_of(&path), opts)
143 .map(Self)
144 }
145
146 /// The engine store underneath.
147 #[must_use]
148 pub fn engine(&self) -> &store_io_engine::Store<NativePlatform> {
149 &self.0
150 }
151}
152
153impl Deref for Store {
154 type Target = store_io_engine::Store<NativePlatform>;
155
156 fn deref(&self) -> &Self::Target {
157 &self.0
158 }
159}
160
161impl From<store_io_engine::Store<NativePlatform>> for Store {
162 fn from(s: store_io_engine::Store<NativePlatform>) -> Self {
163 Self(s)
164 }
165}
166
167impl From<Store> for store_io_engine::Store<NativePlatform> {
168 fn from(s: Store) -> Self {
169 s.0
170 }
171}
172
173/// A directory of small files, each replaced atomically and durably (see
174/// [`store_io_engine::Directory`]).
175///
176/// ```no_run
177/// use store_io::Directory;
178///
179/// let dir = Directory::open("data/meta", true)?;
180/// dir.replace("CURRENT", b"MANIFEST-000042")?;
181/// assert_eq!(dir.read("CURRENT")?, b"MANIFEST-000042");
182/// # Ok::<(), store_io::Error>(())
183/// ```
184#[derive(Debug)]
185pub struct Directory(store_io_engine::Directory<NativePlatform>);
186
187impl Directory {
188 /// Opens `path` (creating it if `create`), with default trust.
189 ///
190 /// # Errors
191 ///
192 /// `NotFound`, `Io`.
193 pub fn open(path: impl AsRef<Path>, create: bool) -> Result<Self, Error> {
194 Self::open_with(path, create, Trust::default())
195 }
196
197 /// Opens `path` with explicit trust inputs.
198 ///
199 /// # Errors
200 ///
201 /// As [`Self::open`].
202 pub fn open_with(path: impl AsRef<Path>, create: bool, trust: Trust) -> Result<Self, Error> {
203 store_io_engine::Directory::open(platform(LinuxTier::Auto)?, path_of(&path), create, trust)
204 .map(Self)
205 }
206
207 /// The largest object a handle writes or reads by default (see
208 /// [`Self::with_max_object`]): 16 MiB.
209 pub const DEFAULT_MAX_OBJECT: u64 =
210 store_io_engine::Directory::<NativePlatform>::DEFAULT_MAX_OBJECT;
211
212 /// The handle with `max` as its largest object, written or read (see
213 /// [`store_io_engine::Directory::with_max_object`]): a larger `replace`
214 /// is refused before anything is written, and a larger file by `read`
215 /// before anything is allocated.
216 #[must_use]
217 pub fn with_max_object(self, max: u64) -> Self {
218 Self(self.0.with_max_object(max))
219 }
220}
221
222impl Deref for Directory {
223 type Target = store_io_engine::Directory<NativePlatform>;
224
225 fn deref(&self) -> &Self::Target {
226 &self.0
227 }
228}
229
230fn io(op: Op, raw: OsError) -> Error {
231 Error::Io {
232 op,
233 raw,
234 ctx: ErrorContext::default(),
235 }
236}
237
238/// Copies the store in `from` into a new store (container format 2) in
239/// `to`, whose directory must hold no store: the way a format-1 store
240/// moves to format 2, and the way to compact a format-2 one. The source is
241/// opened read-only and never written; the copy is built under a temporary
242/// name and renamed into place only when whole and durable, so after a
243/// crash `to` holds nothing or the whole copy (run the copy again).
244///
245/// ```no_run
246/// use store_io::CopyOptions;
247///
248/// let r = store_io::copy_store("data/orders", "data/orders-2", CopyOptions::default())?;
249/// println!("{} regions, {} bytes", r.regions, r.bytes);
250/// # Ok::<(), store_io::Error>(())
251/// ```
252///
253/// # Errors
254///
255/// As [`store_io_engine::copy_store`], among them: `AlreadyExists` if `to`
256/// holds a store; `Locked` while a writer holds `from` open or another
257/// copy into `to` runs;
258/// `NotWritten(NotReady | Frozen | TooLarge)` for a source it cannot copy
259/// whole, before anything is created; `NotWritten(Misaligned)` if `to`'s
260/// device needs larger blocks than the source's; `NotFound`, `Corruption`
261/// or `Unsupported` opening the source; `UnsafeDevice`, `Unverified` (the
262/// target's device refused: see `StoreOptions::trust`), `NoSpace`, `Io`;
263/// `DurabilityUnknown` if the final rename or directory flush failed (the
264/// copy may or may not be in place).
265pub fn copy_store(
266 from: impl AsRef<Path>,
267 to: impl AsRef<Path>,
268 opts: CopyOptions,
269) -> Result<CopyReport, Error> {
270 store_io_engine::copy_store(
271 platform(opts.store.linux_tier)?,
272 path_of(&from),
273 path_of(&to),
274 opts,
275 )
276}
277
278/// A sink writing an export into the existing, empty directory `dir`: one
279/// data file per region and a `regions.info` record, restorable only once
280/// the export returned `Ok`.
281///
282/// ```no_run
283/// let store = store_io::Store::open("data/orders")?;
284/// let mut sink = store_io::file_sink("backup/orders", store_io::FileSinkOptions::default())?;
285/// let report = store.export(&mut sink).map_err(|e| e.error)?;
286/// assert!(report.complete);
287/// # Ok::<(), store_io::Error>(())
288/// ```
289///
290/// # Errors
291///
292/// As [`store_io_engine::FileSink::create`]: `NotFound` if `dir` does not
293/// exist, `NotWritten(InvalidOptions)` if it is not empty, `Io`.
294pub fn file_sink(dir: impl AsRef<Path>, opts: FileSinkOptions) -> Result<FileSink, Error> {
295 store_io_engine::FileSink::create(platform(opts.store.linux_tier)?, path_of(&dir), opts)
296}
297
298/// A source reading back the export a [`file_sink`] wrote into `dir`, for
299/// [`import`]: every span of every data file checked against its CRC-32C
300/// before its bytes are handed out.
301///
302/// # Errors
303///
304/// As [`store_io_engine::FileSource::open`]: `NotFound` if `dir` does not
305/// exist, `Corruption` if it holds no `regions.info`, `Io`.
306pub fn file_source(dir: impl AsRef<Path>, opts: FileSourceOptions) -> Result<FileSource, Error> {
307 store_io_engine::FileSource::open(platform(opts.store.linux_tier)?, path_of(&dir), opts)
308}
309
310/// Imports an export into a new store in `to`, whose directory must hold
311/// no store: built under a temporary name and renamed into place only when
312/// whole and durable, so after a crash `to` holds nothing or the whole
313/// store (import again).
314///
315/// ```no_run
316/// let mut source = store_io::file_source("backup/orders", Default::default())?;
317/// let r = store_io::import(&mut source, "data/orders", store_io::ImportOptions::default())
318/// .map_err(|e| e.error)?;
319/// println!("{} regions, {} bytes", r.regions.len(), r.bytes);
320/// # Ok::<(), store_io::Error>(())
321/// ```
322///
323/// # Errors
324///
325/// As [`store_io_engine::import`] (boxed, with how far it got).
326pub fn import(
327 source: &mut dyn ImportSource,
328 to: impl AsRef<Path>,
329 opts: ImportOptions,
330) -> Result<ImportReport, Box<ImportError>> {
331 let p = platform(opts.store.linux_tier)
332 .map_err(|error| Box::new(ImportError::from_error(error)))?;
333 store_io_engine::import(p, path_of(&to), source, opts)
334}
335
336/// A sink restoring an export straight into a new store in `to`, whose
337/// directory must hold no store (its copy lock held until the sink drops):
338/// each region rebuilt as [`import`] rebuilds it, the store renamed into
339/// place only once the export returned `Ok`, so after a crash `to` holds
340/// nothing or the whole store.
341///
342/// ```no_run
343/// let store = store_io::Store::open("data/orders")?;
344/// let mut sink = store_io::store_sink("replica/orders", store_io::StoreSinkOptions::default())?;
345/// let report = store.export(&mut sink).map_err(|e| e.error)?;
346/// assert!(report.complete && sink.is_complete());
347/// # Ok::<(), store_io::Error>(())
348/// ```
349///
350/// # Errors
351///
352/// As [`store_io_engine::StoreSink::create`]: `AlreadyExists` if a store
353/// is in `to`, `Locked` while another copy, import or sink into it runs,
354/// `Io`.
355pub fn store_sink(to: impl AsRef<Path>, opts: StoreSinkOptions) -> Result<StoreSink, Error> {
356 store_io_engine::StoreSink::create(platform(opts.store.linux_tier)?, path_of(&to), opts)
357}
358
359/// Probes the device under the existing directory `dir`: the evidence
360/// store-io reads and the durability class it would decide with default
361/// trust. Creates, probes and removes a small temporary file there.
362///
363/// ```no_run
364/// let p = store_io::probe("data")?;
365/// println!("{:?}: {:?}", p.decision.class, p.decision.reasons);
366/// # Ok::<(), store_io::Error>(())
367/// ```
368///
369/// # Errors
370///
371/// `NotFound`, `Io`.
372pub fn probe(dir: impl AsRef<Path>) -> Result<Probe, Error> {
373 let p = NativePlatform::new();
374 let d = p.open_dir(path_of(&dir), false).map_err(|raw| {
375 if store_io_core::errno::is_not_found(raw) {
376 Error::NotFound { what: Named::Store }
377 } else {
378 io(Op::Open, raw)
379 }
380 })?;
381 // Unique per call (process, a counter and a random part, for processes
382 // in different PID namespaces sharing a directory), so concurrent
383 // probes of one directory never share, or remove, each other's file.
384 static PROBES: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
385 let n = PROBES.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
386 let salt = {
387 use std::hash::{BuildHasher, Hasher};
388 let mut h = std::collections::hash_map::RandomState::new().build_hasher();
389 h.write_u64(n);
390 h.finish()
391 };
392 let unique = format!(".store-io-probe-{}-{n}-{salt:016x}", std::process::id());
393 let name = FileName::new(&unique).map_err(|_| Error::NotWritten {
394 cause: NotWrittenCause::InvalidName,
395 ctx: ErrorContext::default(),
396 })?;
397 let file = p
398 .create_file(&d, &name)
399 .map_err(|raw| io(Op::Create, raw))?;
400 let evidence = p
401 .allocate(&file, 4096)
402 .and_then(|()| p.probe(&d, &file))
403 .map_err(|raw| io(Op::Probe, raw));
404 drop(file);
405 let _removed = p.unlink(&d, &name);
406 let evidence = evidence?;
407 let decision = decide(&evidence, &Trust::default());
408 Ok(Probe { evidence, decision })
409}