Skip to main content

store_io_engine/
export.rs

1//! Streaming export (PLANNING R5.6, D-148): a store's regions, their
2//! attributes and their bytes, pushed in order to a caller's [`ExportSink`],
3//! for backup, restore and replication shipping. How the stream travels is
4//! the caller's: store-io does not frame it.
5//!
6//! **One plan, then each region at its turn.** Export shares `copy_store`'s
7//! plan (`copy.rs`): every refusal and every volume fact is decided once,
8//! before the sink is called, so the two cannot drift. Each region is then
9//! looked up again at its turn: it must still be the planned (id,
10//! generation), else the export fails (`StaleGeneration` if recycled,
11//! `NotReady` if released), the region named. Sizes are fixed at the plan:
12//! a page region grown since is exported at its planned size, a positioned
13//! append region's durable prefix is clamped to its planned data size, and
14//! a slot's object is read at its turn and refused `TooLarge` as the plan
15//! refuses one.
16//!
17//! **What each kind exports.** A positioned append region its durable
18//! prefix; an unpositioned one (any reopened store) its whole data area, as
19//! read, with no durable length; a page region its data area, as read (a
20//! writable store's pages may hold bytes not yet durable: the header says
21//! the store was writable); a slot its committed object, as one chunk; a
22//! region of a stale page (read-only opens only) its whole capacity and the
23//! extra extent its header lists, by container offset.
24//!
25//! **Bytes.** One background read stream for the whole call, paced by the
26//! call's record on the store's device group, its buffers taken before the
27//! sink's `begin` (so the header carries its `lend`), each region added to
28//! it at its turn. Chunks are owned: a sink may hold up to `lend - 1` of
29//! them and drop them later. When the stream needs a buffer and every one
30//! is lent, the sink's `settle` is called once; if it still holds them all,
31//! the export fails `TooManyInFlight`. A media error fails the export at
32//! once (a backup never has a silent hole).
33//!
34//! **What the report says.** The offset reached in the region being
35//! exported counts only bytes the sink's `data` accepted, and with a
36//! reference sink (`FileSink`, `StoreSink`) only bytes it wrote (a chunk it
37//! holds in a piece not yet written counts none); a region counts as
38//! exported only once `end_region` returned `Ok`; the export is `complete`
39//! only once `finish` returned `Ok`.
40
41use std::sync::Arc;
42use std::sync::atomic::Ordering;
43
44use store_io_core::error::{Error, ErrorContext, NotWrittenCause, Op, OsError};
45use store_io_core::id::RegionId;
46use store_io_format::exportinfo::{self, InfoHeader, RECORD_LEN, RegionRecord};
47use store_io_format::meta::{FillPattern, RegionKind, RegionName};
48use store_io_format::meta2::max_object;
49use store_io_platform::Platform;
50
51use crate::call::{self, CallState};
52use crate::copy::{CopyPlan, Facts};
53use crate::io::{ctx_range, not_written};
54use crate::lifecycle::retired_error;
55use crate::slot_handle::committed_object;
56use crate::store::{LiveRegion, Store, StoreOptions};
57use crate::stream::{Chunk, Locator, ReadStream};
58
59/// How [`Store::export_with`] exports.
60#[derive(Debug, Clone, Default)]
61#[non_exhaustive]
62pub struct ExportOptions {
63    /// The regions to export, by name, each once (`None`: every region, in
64    /// the store's order). A subset is exported in the store's order; the
65    /// volume-wide refusals (a frozen store, lost or stale pages,
66    /// unavailable entries) stand whatever it names.
67    pub regions: Option<Vec<String>>,
68    /// The export stream's reads in flight, 1 to
69    /// [`MAX_DEPTH`](crate::MAX_DEPTH) (`None`: the stream's default, as
70    /// [`StreamOptions`](crate::StreamOptions)).
71    pub depth: Option<usize>,
72    /// Export a store with stale or lost table pages or unavailable
73    /// entries as far as it can be, as
74    /// [`CopyOptions::accept_stale_pages`](crate::CopyOptions::accept_stale_pages):
75    /// what is left out is in the header's [`VolumeFacts`]. A region of a
76    /// stale page is exported in full, and only from a read-only open.
77    pub accept_stale_pages: bool,
78}
79
80/// What the plan found about the volume as a whole (the facts
81/// [`CopyReport`](crate::CopyReport) carries), so a restore never hides a
82/// loss.
83#[derive(Debug, Clone, Default, PartialEq, Eq)]
84#[non_exhaustive]
85pub struct VolumeFacts {
86    /// Entries left out (unavailable, or of a kind this version does not
87    /// know), in id order.
88    pub omitted: Vec<RegionId>,
89    /// Table pages lost: their regions are unknown.
90    pub lost_pages: Vec<u32>,
91    /// Table pages stale: their regions exported in full.
92    pub stale_pages: Vec<u32>,
93    /// Each region exported in full, with the data size its entry records
94    /// (the bytes past it are its capacity's, as read).
95    pub verified_up_to: Vec<(String, u64)>,
96    /// Regions possibly superseded (another region's newer footprint may
97    /// cover their bytes).
98    pub superseded: Vec<String>,
99}
100
101/// What a sink's [`ExportSink::begin`] is told before the first region.
102#[derive(Debug, Clone, PartialEq, Eq)]
103#[non_exhaustive]
104pub struct ExportHeader {
105    /// Regions that follow, exactly.
106    pub region_count: u32,
107    /// The store's block size: every region's data size is a multiple.
108    pub block_size: u64,
109    /// The volume's facts.
110    pub facts: VolumeFacts,
111    /// Whether the store was open for writing: a page region's bytes are
112    /// then as read, possibly not yet durable.
113    pub writable: bool,
114    /// Chunks the export's stream lends at once: a sink holding `lend - 1`
115    /// at most never meets `TooManyInFlight` by its own counting.
116    pub lend: usize,
117    /// An upper bound on the bytes a restore into a new store of the
118    /// default geometry takes: its metadata, two header blocks per region,
119    /// each region's data size (a capped region's cap when larger), each
120    /// slot at the largest object a format-2 slot holds.
121    pub total: u64,
122}
123
124/// One region, as a sink's [`ExportSink::region`] is told before its bytes.
125#[derive(Debug, Clone, PartialEq, Eq)]
126#[non_exhaustive]
127pub struct RegionInfo {
128    /// Its name.
129    pub name: String,
130    /// Its kind.
131    pub kind: RegionKind,
132    /// Its generation (a format-1 slot's: 1).
133    pub generation: u64,
134    /// Its space tag.
135    pub tag: u32,
136    /// Its growth cap, if capped.
137    pub cap: Option<u64>,
138    /// How its blocks were filled at provisioning.
139    pub fill: FillPattern,
140    /// The keyed fill's key.
141    pub fill_key: [u8; 16],
142    /// The store's block size.
143    pub block_size: u64,
144    /// Its data size as planned (a region exported in full: the length of
145    /// what it exports).
146    pub data_size: u64,
147    /// A positioned append region's durable length as exported, at most
148    /// `data_size` (`None`: an unpositioned append region's whole data
149    /// area, as read, or another kind).
150    pub durable_len: Option<u64>,
151    /// A slot's committed object's length, `Some(0)` for an empty one
152    /// (`None`: none committed, or another kind).
153    pub object_len: Option<u64>,
154    /// Exported in full (a region of a stale page).
155    pub full_copy: bool,
156    /// Possibly superseded.
157    pub superseded: bool,
158}
159
160impl RegionInfo {
161    /// The bytes its region's data in the stream holds: a slot's object's
162    /// length, an append region's durable length when it has one, else its
163    /// data size.
164    #[must_use]
165    pub fn stream_len(&self) -> u64 {
166        match self.kind {
167            RegionKind::Slot => self.object_len.unwrap_or(0),
168            _ => self.durable_len.unwrap_or(self.data_size),
169        }
170    }
171
172    /// Whether a restore into a store of blocks of `block_size` can take
173    /// it as it says, checked as untrusted input: a valid name, a non-zero
174    /// generation, the block size the target's (a power of two from 512 to
175    /// 65,536), a data size and cap in whole blocks with the cap at least
176    /// the data size, a durable length only on an append region not
177    /// exported in full and at most its data size, an object length only
178    /// on a slot and at most what a slot holds, no slot exported in full,
179    /// and its restored footprint (two header blocks and its data) within
180    /// `u64`.
181    ///
182    /// # Errors
183    ///
184    /// `NotWritten(InvalidOptions)` at the first check that fails.
185    pub fn validate(&self, block_size: u64) -> Result<(), Error> {
186        let bad = || not_written(NotWrittenCause::InvalidOptions, ErrorContext::default());
187        if RegionName::new(&self.name).is_err() || self.generation == 0 {
188            return Err(bad());
189        }
190        let block = self.block_size;
191        if block != block_size || !block.is_power_of_two() || !(512..=1 << 16).contains(&block) {
192            return Err(bad());
193        }
194        if self.data_size % block != 0 {
195            return Err(bad());
196        }
197        if let Some(cap) = self.cap {
198            if cap < self.data_size || cap % block != 0 {
199                return Err(bad());
200            }
201        }
202        let slot = self.kind == RegionKind::Slot;
203        if let Some(d) = self.durable_len {
204            if self.kind != RegionKind::Append || self.full_copy || d > self.data_size {
205                return Err(bad());
206            }
207        }
208        if let Some(n) = self.object_len {
209            let most = max_object(block.trailing_zeros() as u8) as u64;
210            if !slot || n > most {
211                return Err(bad());
212            }
213        }
214        if self.full_copy && slot {
215            return Err(bad());
216        }
217        let footprint = block
218            .checked_mul(2)
219            .and_then(|h| h.checked_add(self.cap.unwrap_or(0).max(self.data_size)));
220        if footprint.is_none() {
221            return Err(bad());
222        }
223        Ok(())
224    }
225
226    /// The format's record of it (`None` if its name is not a region name
227    /// or its object length does not fit the record).
228    pub(crate) fn to_record(&self) -> Option<RegionRecord> {
229        Some(RegionRecord {
230            name: RegionName::new(&self.name).ok()?,
231            kind: self.kind,
232            fill: self.fill,
233            fill_key: self.fill_key,
234            tag: self.tag,
235            generation: self.generation,
236            cap: self.cap,
237            data_size: self.data_size,
238            durable_len: self.durable_len,
239            object_len: match self.object_len {
240                Some(n) => Some(u32::try_from(n).ok()?),
241                None => None,
242            },
243            stream_len: self.stream_len(),
244            full_copy: self.full_copy,
245            superseded: self.superseded,
246        })
247    }
248
249    /// The region a record describes, in a stream of blocks of
250    /// `block_size`.
251    pub(crate) fn from_record(rec: &RegionRecord, block_size: u64) -> Self {
252        Self {
253            name: rec.name.as_str().to_owned(),
254            kind: rec.kind,
255            generation: rec.generation,
256            tag: rec.tag,
257            cap: rec.cap,
258            fill: rec.fill,
259            fill_key: rec.fill_key,
260            block_size,
261            data_size: rec.data_size,
262            durable_len: rec.durable_len,
263            object_len: rec.object_len.map(u64::from),
264            full_copy: rec.full_copy,
265            superseded: rec.superseded,
266        }
267    }
268
269    /// Its bytes in format `exportinfo`'s region record ([`RECORD_LEN`],
270    /// versioned with the record), for a caller's own framing; the block
271    /// size travels in the header ([`ExportHeader::to_bytes`]).
272    ///
273    /// # Errors
274    ///
275    /// `NotWritten(InvalidOptions)` if its fields cannot be a record (a
276    /// name not a region name, an object length past `u32`, fields that
277    /// disagree with its kind).
278    pub fn to_bytes(&self) -> Result<[u8; RECORD_LEN], Error> {
279        let bad = || not_written(NotWrittenCause::InvalidOptions, ErrorContext::default());
280        let rec = self.to_record().ok_or_else(bad)?;
281        rec.check().map_err(|_| bad())?;
282        Ok(rec.encode())
283    }
284
285    /// A region from bytes [`Self::to_bytes`] wrote, in a stream of blocks
286    /// of `block_size`, decoded and then [`Self::validate`]d (the bytes are
287    /// untrusted).
288    ///
289    /// # Errors
290    ///
291    /// `NotWritten(InvalidOptions)` if the bytes are not a record or the
292    /// region fails [`Self::validate`].
293    pub fn from_bytes(bytes: &[u8], block_size: u64) -> Result<Self, Error> {
294        let rec = RegionRecord::decode(bytes)
295            .map_err(|_| not_written(NotWrittenCause::InvalidOptions, ErrorContext::default()))?;
296        let info = Self::from_record(&rec, block_size);
297        info.validate(block_size)?;
298        Ok(info)
299    }
300}
301
302impl ExportHeader {
303    /// Its bytes: format `exportinfo`'s header and lists, sealed by a
304    /// CRC-32C, for a caller's own framing (the regions follow as
305    /// [`RegionInfo::to_bytes`] records).
306    ///
307    /// # Errors
308    ///
309    /// `NotWritten(InvalidOptions)` if it cannot be encoded: a block size
310    /// out of range, a verified name not a region name, a list holding an
311    /// item twice, or a count past `u32`.
312    pub fn to_bytes(&self) -> Result<Vec<u8>, Error> {
313        let bad = || not_written(NotWrittenCause::InvalidOptions, ErrorContext::default());
314        // A power of two (its range the codec's to check).
315        let block = self.block_size;
316        if !block.is_power_of_two() {
317            return Err(bad());
318        }
319        let mut omitted: Vec<u64> = self.facts.omitted.iter().map(|id| id.get()).collect();
320        omitted.sort_unstable();
321        let mut lost = self.facts.lost_pages.clone();
322        lost.sort_unstable();
323        let mut stale = self.facts.stale_pages.clone();
324        stale.sort_unstable();
325        let mut verified = Vec::with_capacity(self.facts.verified_up_to.len());
326        for (name, len) in &self.facts.verified_up_to {
327            verified.push((RegionName::new(name).map_err(|_| bad())?, *len));
328        }
329        verified.sort_by_key(|(n, _)| n.as_padded());
330        let h = InfoHeader {
331            writable: self.writable,
332            block_log2: block.trailing_zeros() as u8,
333            region_count: self.region_count,
334            lend: u32::try_from(self.lend).unwrap_or(u32::MAX),
335            total: self.total,
336            omitted_count: 0,
337            lost_count: 0,
338            stale_count: 0,
339            verified_count: 0,
340        };
341        let counted = InfoHeader {
342            omitted_count: u32::try_from(omitted.len()).map_err(|_| bad())?,
343            lost_count: u32::try_from(lost.len()).map_err(|_| bad())?,
344            stale_count: u32::try_from(stale.len()).map_err(|_| bad())?,
345            verified_count: u32::try_from(verified.len()).map_err(|_| bad())?,
346            ..h
347        };
348        let len = exportinfo::head_len(&counted)
349            .and_then(|n| n.checked_add(4))
350            .ok_or_else(bad)?;
351        let mut out = vec![0u8; len];
352        let n = exportinfo::encode_head(&h, &omitted, &lost, &stale, &verified, &mut out)
353            .map_err(|_| bad())?;
354        out.truncate(n);
355        Ok(out)
356    }
357
358    /// A header from bytes [`Self::to_bytes`] wrote (or a `regions.info`'s
359    /// header and lists, sealed), decoded and checked: the CRC-32C, the
360    /// block size in range, each list in its order. Its `superseded` facts
361    /// are not carried (each region's record says it).
362    ///
363    /// # Errors
364    ///
365    /// `NotWritten(InvalidOptions)` if the bytes are not a sealed header.
366    pub fn from_bytes(bytes: &[u8]) -> Result<Self, Error> {
367        let info = exportinfo::decode_head(bytes)
368            .map_err(|_| not_written(NotWrittenCause::InvalidOptions, ErrorContext::default()))?;
369        Ok(Self::from_view(&info))
370    }
371
372    /// The header a decoded view holds (its lists the volume's facts).
373    pub(crate) fn from_view(info: &exportinfo::ExportInfo<'_>) -> Self {
374        let h = info.header;
375        Self {
376            region_count: h.region_count,
377            block_size: 1u64 << h.block_log2,
378            facts: VolumeFacts {
379                omitted: info.omitted().map(RegionId::from_raw).collect(),
380                lost_pages: info.lost_pages().collect(),
381                stale_pages: info.stale_pages().collect(),
382                verified_up_to: info
383                    .verified()
384                    .map(|(n, len)| (n.as_str().to_owned(), len))
385                    .collect(),
386                superseded: Vec::new(),
387            },
388            writable: h.writable,
389            lend: h.lend as usize,
390            total: h.total,
391        }
392    }
393}
394
395/// Where an import pulls a store from: the header an export's sink was
396/// given, then each region's info and its bytes, in order. A caller's own
397/// source (its framing carried by [`ExportHeader::to_bytes`] and
398/// [`RegionInfo::to_bytes`]) or a [`FileSource`](crate::FileSource).
399/// Everything it yields is untrusted: the import checks it.
400pub trait ImportSource {
401    /// The export's header (first, once).
402    ///
403    /// # Errors
404    ///
405    /// The source's: the import stops, nothing renamed.
406    fn header(&mut self) -> Result<ExportHeader, SinkError>;
407
408    /// The next region's info, or `None` after the last.
409    ///
410    /// # Errors
411    ///
412    /// The source's: the import stops, nothing renamed.
413    fn next_region(&mut self) -> Result<Option<RegionInfo>, SinkError>;
414
415    /// The current region's next bytes into `buf` (at most `buf.len()`):
416    /// how many, 0 at the region's end.
417    ///
418    /// # Errors
419    ///
420    /// The source's: the import stops, nothing renamed.
421    fn read(&mut self, buf: &mut [u8]) -> Result<usize, SinkError>;
422}
423
424/// How far an export got.
425#[derive(Debug, Clone, Default, PartialEq, Eq)]
426#[non_exhaustive]
427pub struct ExportReport {
428    /// Regions exported: each counted once its `end_region` returned `Ok`.
429    pub regions: usize,
430    /// Bytes the sink's `data` accepted.
431    pub bytes: u64,
432    /// The region being exported when the export stopped (`None` once
433    /// every region ended, or before the first).
434    pub region: Option<String>,
435    /// The offset in that region's stream the sink accepted bytes up to.
436    pub offset: u64,
437    /// Whether the store was open for writing.
438    pub writable: bool,
439    /// Every region ended and the sink's `finish` returned `Ok`.
440    pub complete: bool,
441    /// The volume's facts (empty if the plan refused).
442    pub facts: VolumeFacts,
443}
444
445/// A sink's refusal, without payload: an I/O error, a refusal, or the
446/// caller's own code.
447#[derive(Debug, Clone, Copy, PartialEq, Eq)]
448#[non_exhaustive]
449pub enum SinkError {
450    /// An I/O error, as the operating system reported it.
451    Io(OsError),
452    /// The sink refused the call (out of order, or past what it was told).
453    Refused,
454    /// The caller's own code.
455    Other(u32),
456}
457
458impl core::fmt::Display for SinkError {
459    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
460        match self {
461            Self::Io(raw) => write!(f, "sink I/O error {}", raw.code),
462            Self::Refused => f.write_str("sink refused the call"),
463            Self::Other(code) => write!(f, "sink error {code}"),
464        }
465    }
466}
467
468impl std::error::Error for SinkError {}
469
470/// An export that stopped: why, the sink's own error if it was the sink's
471/// (`error` is then `Io` for [`SinkError::Io`], `NotWritten(Interrupted)`
472/// otherwise), and how far it got (`complete` false).
473#[derive(Debug, Clone)]
474#[non_exhaustive]
475pub struct ExportError {
476    /// Why it stopped (the region in its context when one was being
477    /// exported).
478    pub error: Error,
479    /// The sink's error, if the sink stopped it.
480    pub sink: Option<SinkError>,
481    /// How far it got.
482    pub report: ExportReport,
483}
484
485impl core::fmt::Display for ExportError {
486    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
487        write!(f, "export stopped: {}", self.error)
488    }
489}
490
491impl std::error::Error for ExportError {
492    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
493        Some(&self.error)
494    }
495}
496
497/// Where an export pushes a store: its header, then each region's info,
498/// its bytes in order and its end, then the finish. Pacing is invisible to
499/// it: the export's reads are paced by its own call.
500pub trait ExportSink {
501    /// Before the first region.
502    ///
503    /// # Errors
504    ///
505    /// The sink's: the export stops.
506    fn begin(&mut self, header: &ExportHeader) -> Result<(), SinkError>;
507
508    /// Before a region's bytes.
509    ///
510    /// # Errors
511    ///
512    /// The sink's: the export stops.
513    fn region(&mut self, info: &RegionInfo) -> Result<(), SinkError>;
514
515    /// The region's next bytes, contiguous from offset 0 of its stream
516    /// ([`Chunk::offset`]). The sink owns the chunk: it may keep up to the
517    /// header's `lend - 1` and drop them later.
518    ///
519    /// # Errors
520    ///
521    /// The sink's: the export stops; the report's offset is where this
522    /// chunk began.
523    fn data(&mut self, chunk: Chunk) -> Result<(), SinkError>;
524
525    /// Called when the export needs a buffer and the sink holds every one
526    /// lent: drop the chunks no longer needed. If it still holds them all
527    /// the export fails `TooManyInFlight`. By default, nothing.
528    ///
529    /// # Errors
530    ///
531    /// The sink's: the export stops.
532    fn settle(&mut self) -> Result<(), SinkError> {
533        Ok(())
534    }
535
536    /// After a region's last bytes: the region counts as exported once
537    /// this returns `Ok`.
538    ///
539    /// # Errors
540    ///
541    /// The sink's: the export stops.
542    fn end_region(&mut self) -> Result<(), SinkError>;
543
544    /// After the last region (right after `begin` with none): the export is
545    /// complete once this returns `Ok`.
546    ///
547    /// # Errors
548    ///
549    /// The sink's: the export stops, incomplete.
550    fn finish(&mut self) -> Result<(), SinkError>;
551}
552
553/// Why an export stopped, before the report is attached.
554struct Stop {
555    error: Error,
556    sink: Option<SinkError>,
557}
558
559impl From<Error> for Stop {
560    fn from(error: Error) -> Self {
561        Self { error, sink: None }
562    }
563}
564
565/// A refusal naming `region`.
566fn refused(cause: NotWrittenCause, region: Option<RegionId>) -> Error {
567    Error::NotWritten {
568        cause,
569        ctx: ErrorContext {
570            region,
571            ..ErrorContext::default()
572        },
573    }
574}
575
576/// The sink's error as the export's, naming `region`.
577fn sunk(region: Option<RegionId>) -> impl Fn(SinkError) -> Stop {
578    move |e| {
579        let ctx = ErrorContext {
580            region,
581            ..ErrorContext::default()
582        };
583        let error = match e {
584            SinkError::Io(raw) => Error::Io {
585                op: Op::Write,
586                raw,
587                ctx,
588            },
589            SinkError::Refused | SinkError::Other(_) => Error::NotWritten {
590                cause: NotWrittenCause::Interrupted,
591                ctx,
592            },
593        };
594        Stop {
595            error,
596            sink: Some(e),
597        }
598    }
599}
600
601fn too_many(e: &Error) -> bool {
602    matches!(
603        e,
604        Error::NotWritten {
605            cause: NotWrittenCause::TooManyInFlight,
606            ..
607        }
608    )
609}
610
611impl<P: Platform> Store<P>
612where
613    P::Queue: Send,
614{
615    /// Exports every region to `sink`, in the store's order: see the
616    /// module docs and [`Self::export_with`].
617    ///
618    /// # Errors
619    ///
620    /// As [`Self::export_with`].
621    pub fn export(&self, sink: &mut dyn ExportSink) -> Result<ExportReport, Box<ExportError>> {
622        self.export_with(sink, ExportOptions::default())
623    }
624
625    /// Exports the regions `opts` names (all by default) to `sink`: its
626    /// `begin` with the header, then for each region its `region`, its
627    /// bytes through `data` and its `end_region`, then its `finish`.
628    ///
629    /// # Errors
630    ///
631    /// An [`ExportError`] (boxed: it holds the report) with how far it got. Before the sink is
632    /// called: `NotWritten(InvalidOptions)` for a region named twice, a
633    /// name no region has, a depth out of range, or on a writable open a
634    /// region that only a read-only open exports (one of a stale page,
635    /// possibly superseded, or at risk: open the store read-only to export
636    /// it); `NotWritten(NotReady)` for an unavailable region or an entry
637    /// that does not decode, `NotWritten(Frozen)` for a stale or lost table
638    /// page (both unless [`ExportOptions::accept_stale_pages`]);
639    /// `NotWritten(TooLarge)` for a slot object a format-2 slot cannot
640    /// hold; `NotWritten(StaleGeneration)` for a region that changed while
641    /// the plan read it; `NotWritten(PoolExhausted)` with too few buffers
642    /// free for the stream; `Io` if its queue could not be made. At a
643    /// region's turn: `NotWritten(StaleGeneration)` (recycled) or
644    /// `NotWritten(NotReady)` (released) since the plan or while it
645    /// streams, `NotWritten(TooLarge)` for a slot object committed since
646    /// that a format-2 slot cannot hold; a read's errors as a
647    /// [`ReadStream`]'s, a media error (`Corruption(MediaError)`, the
648    /// range named) among them; `NotWritten(TooManyInFlight)` if the sink
649    /// still holds every lent chunk after `settle`. Any error of the sink
650    /// (in `sink`).
651    pub fn export_with(
652        &self,
653        sink: &mut dyn ExportSink,
654        opts: ExportOptions,
655    ) -> Result<ExportReport, Box<ExportError>> {
656        let mut report = ExportReport {
657            writable: !self.inner.read_only,
658            ..ExportReport::default()
659        };
660        match self.export_run(sink, &opts, &mut report) {
661            Ok(()) => {
662                report.region = None;
663                report.offset = 0;
664                report.complete = true;
665                Ok(report)
666            }
667            Err(Stop { error, sink }) => Err(Box::new(ExportError {
668                error,
669                sink,
670                report,
671            })),
672        }
673    }
674
675    /// The export's plan: the options checked, `copy_store`'s plan of the
676    /// regions named, and on a writable open the regions only a read-only
677    /// open exports refused.
678    fn export_plan(&self, opts: &ExportOptions) -> Result<CopyPlan, Error> {
679        // A name no region has is refused by the plan; the depth by the
680        // stream, before the sink's `begin`.
681        if let Some(names) = &opts.regions {
682            if (1..names.len()).any(|i| names[..i].contains(&names[i])) {
683                return Err(not_written(
684                    NotWrittenCause::InvalidOptions,
685                    ErrorContext::default(),
686                ));
687            }
688        }
689        let plan = self.plan_of(opts.accept_stale_pages, opts.regions.as_deref())?;
690        // A region of a stale page is read by container offset: on a
691        // writable open an accepted rollback may free its extra extent
692        // under it. A superseded or at-risk region may be made writable
693        // and stream newer bytes. Only a read-only open exports them.
694        if !self.inner.read_only {
695            if let Some(f) = plan.facts.iter().find(|f| f.full.is_some() || f.superseded) {
696                return Err(refused(NotWrittenCause::InvalidOptions, Some(f.source.id)));
697            }
698        }
699        Ok(plan)
700    }
701
702    /// An upper bound on what a restore of `facts` takes in a new store of
703    /// the default geometry with blocks of `block` (saturating).
704    fn export_total(facts: &[Facts], block: u64) -> u64 {
705        let defaults = StoreOptions::default();
706        let log2_block = block.trailing_zeros() as u8;
707        let log2_table = defaults.log2_table_slot.clamp(log2_block, 16);
708        let count = u32::try_from(facts.len()).unwrap_or(u32::MAX);
709        let table =
710            crate::v2::layout_for(log2_block, log2_table, defaults.region_capacity.max(count))
711                .data_start();
712        facts.iter().fold(table, |total, f| {
713            let data = match f.kind {
714                RegionKind::Slot => max_object(log2_block) as u64,
715                _ => f.cap.unwrap_or(0).max(f.data_size),
716            };
717            total.saturating_add(2 * block).saturating_add(data)
718        })
719    }
720
721    fn export_run(
722        &self,
723        sink: &mut dyn ExportSink,
724        opts: &ExportOptions,
725        report: &mut ExportReport,
726    ) -> Result<(), Stop> {
727        let plan = self.export_plan(opts)?;
728        // The call's record on this store's device group: the stream's
729        // reads are its pieces; a reference sink joins it at `begin`.
730        let pace = self.inner.domain.new_pace();
731        let call = CallState::new(Arc::clone(&self.inner.qos), pace.clone());
732        let _bound = call::bind(&call);
733        let ran = self.export_body(sink, opts, report, &plan, pace, &call);
734        // A part's failure (a reference sink's write, even one a caller's
735        // wrapper swallowed) is the export's: its error, and the report
736        // clamped to where the failed write began.
737        let failure = call.failure();
738        // To where the failure was in the export's own order of regions:
739        // the regions ended before it, the region and offset when inside
740        // one, and the bytes up to there.
741        let clamp = |report: &mut ExportReport| {
742            if let Some(f) = &failure {
743                // The regions ended before it (a backstop beside the count
744                // of regions every bound sink ended, below: a part that
745                // records a failure without binding as a sink).
746                report.regions = report.regions.min(f.region as usize);
747                let base = call.base(f.region);
748                match plan.facts.get(f.region as usize).filter(|_| f.inside) {
749                    Some(r) => {
750                        // Never past the bytes `data` accepted when the
751                        // export stopped in the failure's region (a write
752                        // that failed inside a later chunk's call began
753                        // past that chunk's start); a region the export
754                        // went past was accepted whole.
755                        let same = report.region.as_deref() == Some(r.name.as_str());
756                        report.region = Some(r.name.clone());
757                        report.offset = if same {
758                            report.offset.min(f.offset)
759                        } else {
760                            f.offset
761                        };
762                    }
763                    None => {
764                        report.region = None;
765                        report.offset = 0;
766                        if let Some(b) = base {
767                            report.bytes = report.bytes.min(b);
768                        }
769                    }
770                }
771            }
772            // Stopped inside a region, whatever stopped it: no further than
773            // every bound reference sink's writes done (a sink holding
774            // accepted bytes not yet written counts none of them), and the
775            // bytes no further than that offset.
776            let inside = match &failure {
777                Some(f) => f.inside.then_some(f.region),
778                None => call.inside_region(),
779            };
780            if let (Some(i), Some(_)) = (inside, report.region.as_ref()) {
781                if let Some(w) = call.written_up_to(i) {
782                    report.offset = report.offset.min(w);
783                }
784                if let Some(b) = call.base(i) {
785                    report.bytes = report.bytes.min(b.saturating_add(report.offset));
786                }
787            }
788            // A region counts only once every bound reference sink ended
789            // it (a wrapper may have swallowed a sink's failure no part
790            // recorded).
791            if let Some(n) = call.regions_ended() {
792                // Short of where the report stands (the region it names,
793                // else past the regions it counts): it stands at region
794                // `n` instead, the first a sink never ended, with that
795                // region's writes done, and nothing past it.
796                let i = n as usize;
797                let named = report
798                    .region
799                    .as_ref()
800                    .and_then(|name| plan.facts.iter().position(|f| &f.name == name));
801                if i < named.unwrap_or(report.regions) {
802                    report.regions = i;
803                    report.region = plan.facts.get(i).map(|f| f.name.clone());
804                    report.offset = call.written_up_to(n).unwrap_or(0);
805                    if let Some(b) = call.base(n) {
806                        report.bytes = report.bytes.min(b.saturating_add(report.offset));
807                    }
808                }
809            }
810        };
811        match ran {
812            Err(mut stop) => {
813                if let Some(f) = &failure {
814                    stop.error = f.error.clone();
815                }
816                clamp(report);
817                Err(stop)
818            }
819            Ok(()) => {
820                if let Some(f) = &failure {
821                    clamp(report);
822                    return Err(Stop {
823                        error: f.error.clone(),
824                        sink: None,
825                    });
826                }
827                // A bound reference sink that never finished (a wrapper
828                // that dropped it, or fell back to another).
829                if !call.all_finished() {
830                    clamp(report);
831                    return Err(Stop::from(not_written(
832                        NotWrittenCause::Interrupted,
833                        ErrorContext::default(),
834                    )));
835                }
836                Ok(())
837            }
838        }
839    }
840
841    /// The export's sink calls, bound to its call.
842    fn export_body(
843        &self,
844        sink: &mut dyn ExportSink,
845        opts: &ExportOptions,
846        report: &mut ExportReport,
847        plan: &CopyPlan,
848        pace: Option<crate::pace::Pace>,
849        call: &CallState,
850    ) -> Result<(), Stop> {
851        report.facts = VolumeFacts {
852            omitted: plan.omitted.clone(),
853            lost_pages: plan.lost_pages.clone(),
854            stale_pages: plan.stale_pages.clone(),
855            verified_up_to: plan.verified_up_to.clone(),
856            superseded: plan.superseded.clone(),
857        };
858        let stream = self.export_stream(opts.depth, pace)?;
859        let header = ExportHeader {
860            region_count: u32::try_from(plan.facts.len()).unwrap_or(u32::MAX),
861            block_size: plan.block,
862            facts: report.facts.clone(),
863            writable: report.writable,
864            lend: stream.lend(),
865            total: Self::export_total(&plan.facts, plan.block),
866        };
867        sink.begin(&header).map_err(sunk(None))?;
868        // A sink that joined the source's record shares its pieces.
869        if call.shares_source() {
870            stream.make_second();
871        }
872        for (i, f) in plan.facts.iter().enumerate() {
873            report.region = Some(f.name.clone());
874            report.offset = 0;
875            let index = u32::try_from(i).unwrap_or(u32::MAX);
876            call.enter(index, report.bytes);
877            self.export_region(f, plan.block, &stream, sink, report)
878                .map_err(|stop| Self::part_error(&stream, stop))?;
879            report.regions += 1;
880            call.leave(index.saturating_add(1));
881        }
882        report.region = None;
883        report.offset = 0;
884        // Every part asked before the export reports anything: a read the
885        // stream lost while a sink's wait collected it is its failure.
886        if let Some(e) = stream.failure() {
887            return Err(Stop::from(e));
888        }
889        // Nothing is in flight: every part was read to its end.
890        stream.cancel()?;
891        sink.finish().map_err(sunk(None))
892    }
893
894    /// The stream's own failure, if a collection for another part of the
895    /// call lost its reads (the sink then only saw its wait end), else
896    /// `stop` as it was.
897    fn part_error(stream: &ReadStream<P>, stop: Stop) -> Stop {
898        match stream.failure() {
899            Some(error) => Stop {
900                error,
901                sink: stop.sink,
902            },
903            None => stop,
904        }
905    }
906
907    /// Slot `r`'s committed object at its turn, read under its header lock
908    /// (a slot is never recycled or released: its gate stays live), refused
909    /// as the plan refuses one a format-2 slot cannot hold.
910    fn object_at_turn(&self, r: &LiveRegion, block: u64) -> Result<Option<Vec<u8>>, Error> {
911        let header = r
912            .header
913            .lock()
914            .unwrap_or_else(std::sync::PoisonError::into_inner)
915            .clone();
916        let object = committed_object(&header, &self.inner.layout);
917        if object
918            .as_ref()
919            .is_some_and(|o| o.len() > max_object(block.trailing_zeros() as u8))
920        {
921            return Err(refused(NotWrittenCause::TooLarge, Some(r.id)));
922        }
923        Ok(object)
924    }
925
926    /// Exports region `f` (of the plan) at its turn.
927    fn export_region(
928        &self,
929        f: &Facts,
930        block: u64,
931        stream: &ReadStream<P>,
932        sink: &mut dyn ExportSink,
933        report: &mut ExportReport,
934    ) -> Result<(), Stop> {
935        // Still the planned (id, generation): a recycle or release closes
936        // the planned region's gate for good (a recycle's new generation is
937        // a region of its own), so a gate still live is the plan's region.
938        let r = &f.source;
939        r.gate.live().map_err(|why| retired_error(why, r))?;
940        let id = Some(r.id);
941        let (durable_len, object, part) = match (f.kind, &f.full) {
942            (RegionKind::Slot, _) => (None, self.object_at_turn(r, block)?, None),
943            // By container offset: its capacity's runs, then its extra
944            // extent.
945            (_, Some((ranges, _))) => (
946                None,
947                None,
948                Some((f.data_size, Locator::Ranges(ranges.clone()))),
949            ),
950            (kind, None) => {
951                // A positioned append region's durable prefix, clamped to
952                // its planned data size (still a valid prefix).
953                let durable = (kind == RegionKind::Append && r.positioned.load(Ordering::Acquire))
954                    .then(|| {
955                        r.frontier
956                            .as_ref()
957                            .map_or(0, crate::frontier::AppendFrontier::durable)
958                            .min(f.data_size)
959                    });
960                (
961                    durable,
962                    None,
963                    Some((durable.unwrap_or(f.data_size), Locator::Map)),
964                )
965            }
966        };
967        let info = RegionInfo {
968            name: f.name.clone(),
969            kind: f.kind,
970            generation: f.attrs.generation,
971            tag: f.attrs.tag,
972            cap: f.cap,
973            fill: f.attrs.fill,
974            fill_key: f.attrs.fill_key,
975            block_size: block,
976            data_size: f.data_size,
977            durable_len,
978            object_len: object.as_ref().map(|o| o.len() as u64),
979            full_copy: f.full.is_some(),
980            superseded: f.superseded,
981        };
982        sink.region(&info).map_err(sunk(id))?;
983        if let Some(o) = object.filter(|o| !o.is_empty()) {
984            let mut settled = false;
985            let chunk = loop {
986                match stream.lend_bytes(r, 0, &o) {
987                    Ok(c) => break c,
988                    Err(e) if too_many(&e) && !settled => {
989                        sink.settle().map_err(sunk(id))?;
990                        settled = true;
991                    }
992                    Err(e) => return Err(e.into()),
993                }
994            };
995            sink.data(chunk).map_err(sunk(id))?;
996            report.offset = o.len() as u64;
997            report.bytes += o.len() as u64;
998        }
999        if let Some((len, loc)) = part {
1000            self.export_bytes(r, len, loc, stream, sink, report)?;
1001        }
1002        sink.end_region().map_err(sunk(id))
1003    }
1004
1005    /// Streams `[0, len)` of `r` (read as `loc` places it) to the sink.
1006    fn export_bytes(
1007        &self,
1008        r: &Arc<LiveRegion>,
1009        len: u64,
1010        loc: Locator,
1011        stream: &ReadStream<P>,
1012        sink: &mut dyn ExportSink,
1013        report: &mut ExportReport,
1014    ) -> Result<(), Stop> {
1015        let id = Some(r.id);
1016        stream.add_part(r, 0..len, loc)?;
1017        let mut got = 0u64;
1018        let mut settled = false;
1019        while got < len {
1020            match stream.pull() {
1021                Some(Ok(c)) => {
1022                    let n = c.bytes().len() as u64;
1023                    // In order from the region's start, with no gap and
1024                    // nothing past its end.
1025                    if n == 0 || c.offset() != got || n > len - got {
1026                        let mut e = crate::store::corrupt();
1027                        if let Error::Corruption { ctx, .. } = &mut e {
1028                            *ctx = ctx_range(r, got, len);
1029                        }
1030                        return Err(e.into());
1031                    }
1032                    settled = false;
1033                    sink.data(c).map_err(sunk(id))?;
1034                    got += n;
1035                    report.offset = got;
1036                    report.bytes += n;
1037                }
1038                Some(Err(e)) if too_many(&e) && !settled => {
1039                    sink.settle().map_err(sunk(id))?;
1040                    settled = true;
1041                }
1042                Some(Err(e)) => return Err(e.into()),
1043                // Ended before the region's bytes did.
1044                None => {
1045                    return Err(
1046                        not_written(NotWrittenCause::Interrupted, ctx_range(r, got, len)).into(),
1047                    );
1048                }
1049            }
1050        }
1051        Ok(())
1052    }
1053}