store_io_engine/export.rs
1//! Streaming export (PLANNING R5.6, D-148): a store's regions, their
2//! attributes and their bytes, pushed in order to a caller's [`ExportSink`],
3//! for backup, restore and replication shipping. How the stream travels is
4//! the caller's: store-io does not frame it.
5//!
6//! **One plan, then each region at its turn.** Export shares `copy_store`'s
7//! plan (`copy.rs`): every refusal and every volume fact is decided once,
8//! before the sink is called, so the two cannot drift. Each region is then
9//! looked up again at its turn: it must still be the planned (id,
10//! generation), else the export fails (`StaleGeneration` if recycled,
11//! `NotReady` if released), the region named. Sizes are fixed at the plan:
12//! a page region grown since is exported at its planned size, a positioned
13//! append region's durable prefix is clamped to its planned data size, and
14//! a slot's object is read at its turn and refused `TooLarge` as the plan
15//! refuses one.
16//!
17//! **What each kind exports.** A positioned append region its durable
18//! prefix; an unpositioned one (any reopened store) its whole data area, as
19//! read, with no durable length; a page region its data area, as read (a
20//! writable store's pages may hold bytes not yet durable: the header says
21//! the store was writable); a slot its committed object, as one chunk; a
22//! region of a stale page (read-only opens only) its whole capacity and the
23//! extra extent its header lists, by container offset.
24//!
25//! **Bytes.** One background read stream for the whole call, paced by the
26//! call's record on the store's device group, its buffers taken before the
27//! sink's `begin` (so the header carries its `lend`), each region added to
28//! it at its turn. Chunks are owned: a sink may hold up to `lend - 1` of
29//! them and drop them later. When the stream needs a buffer and every one
30//! is lent, the sink's `settle` is called once; if it still holds them all,
31//! the export fails `TooManyInFlight`. A media error fails the export at
32//! once (a backup never has a silent hole).
33//!
34//! **What the report says.** The offset reached in the region being
35//! exported counts only bytes the sink's `data` accepted, and with a
36//! reference sink (`FileSink`, `StoreSink`) only bytes it wrote (a chunk it
37//! holds in a piece not yet written counts none); a region counts as
38//! exported only once `end_region` returned `Ok`; the export is `complete`
39//! only once `finish` returned `Ok`.
40
41use std::sync::Arc;
42use std::sync::atomic::Ordering;
43
44use store_io_core::error::{Error, ErrorContext, NotWrittenCause, Op, OsError};
45use store_io_core::id::RegionId;
46use store_io_format::exportinfo::{self, InfoHeader, RECORD_LEN, RegionRecord};
47use store_io_format::meta::{FillPattern, RegionKind, RegionName};
48use store_io_format::meta2::max_object;
49use store_io_platform::Platform;
50
51use crate::call::{self, CallState};
52use crate::copy::{CopyPlan, Facts};
53use crate::io::{ctx_range, not_written};
54use crate::lifecycle::retired_error;
55use crate::slot_handle::committed_object;
56use crate::store::{LiveRegion, Store, StoreOptions};
57use crate::stream::{Chunk, Locator, ReadStream};
58
59/// How [`Store::export_with`] exports.
60#[derive(Debug, Clone, Default)]
61#[non_exhaustive]
62pub struct ExportOptions {
63 /// The regions to export, by name, each once (`None`: every region, in
64 /// the store's order). A subset is exported in the store's order; the
65 /// volume-wide refusals (a frozen store, lost or stale pages,
66 /// unavailable entries) stand whatever it names.
67 pub regions: Option<Vec<String>>,
68 /// The export stream's reads in flight, 1 to
69 /// [`MAX_DEPTH`](crate::MAX_DEPTH) (`None`: the stream's default, as
70 /// [`StreamOptions`](crate::StreamOptions)).
71 pub depth: Option<usize>,
72 /// Export a store with stale or lost table pages or unavailable
73 /// entries as far as it can be, as
74 /// [`CopyOptions::accept_stale_pages`](crate::CopyOptions::accept_stale_pages):
75 /// what is left out is in the header's [`VolumeFacts`]. A region of a
76 /// stale page is exported in full, and only from a read-only open.
77 pub accept_stale_pages: bool,
78}
79
80/// What the plan found about the volume as a whole (the facts
81/// [`CopyReport`](crate::CopyReport) carries), so a restore never hides a
82/// loss.
83#[derive(Debug, Clone, Default, PartialEq, Eq)]
84#[non_exhaustive]
85pub struct VolumeFacts {
86 /// Entries left out (unavailable, or of a kind this version does not
87 /// know), in id order.
88 pub omitted: Vec<RegionId>,
89 /// Table pages lost: their regions are unknown.
90 pub lost_pages: Vec<u32>,
91 /// Table pages stale: their regions exported in full.
92 pub stale_pages: Vec<u32>,
93 /// Each region exported in full, with the data size its entry records
94 /// (the bytes past it are its capacity's, as read).
95 pub verified_up_to: Vec<(String, u64)>,
96 /// Regions possibly superseded (another region's newer footprint may
97 /// cover their bytes).
98 pub superseded: Vec<String>,
99}
100
101/// What a sink's [`ExportSink::begin`] is told before the first region.
102#[derive(Debug, Clone, PartialEq, Eq)]
103#[non_exhaustive]
104pub struct ExportHeader {
105 /// Regions that follow, exactly.
106 pub region_count: u32,
107 /// The store's block size: every region's data size is a multiple.
108 pub block_size: u64,
109 /// The volume's facts.
110 pub facts: VolumeFacts,
111 /// Whether the store was open for writing: a page region's bytes are
112 /// then as read, possibly not yet durable.
113 pub writable: bool,
114 /// Chunks the export's stream lends at once: a sink holding `lend - 1`
115 /// at most never meets `TooManyInFlight` by its own counting.
116 pub lend: usize,
117 /// An upper bound on the bytes a restore into a new store of the
118 /// default geometry takes: its metadata, two header blocks per region,
119 /// each region's data size (a capped region's cap when larger), each
120 /// slot at the largest object a format-2 slot holds.
121 pub total: u64,
122}
123
124/// One region, as a sink's [`ExportSink::region`] is told before its bytes.
125#[derive(Debug, Clone, PartialEq, Eq)]
126#[non_exhaustive]
127pub struct RegionInfo {
128 /// Its name.
129 pub name: String,
130 /// Its kind.
131 pub kind: RegionKind,
132 /// Its generation (a format-1 slot's: 1).
133 pub generation: u64,
134 /// Its space tag.
135 pub tag: u32,
136 /// Its growth cap, if capped.
137 pub cap: Option<u64>,
138 /// How its blocks were filled at provisioning.
139 pub fill: FillPattern,
140 /// The keyed fill's key.
141 pub fill_key: [u8; 16],
142 /// The store's block size.
143 pub block_size: u64,
144 /// Its data size as planned (a region exported in full: the length of
145 /// what it exports).
146 pub data_size: u64,
147 /// A positioned append region's durable length as exported, at most
148 /// `data_size` (`None`: an unpositioned append region's whole data
149 /// area, as read, or another kind).
150 pub durable_len: Option<u64>,
151 /// A slot's committed object's length, `Some(0)` for an empty one
152 /// (`None`: none committed, or another kind).
153 pub object_len: Option<u64>,
154 /// Exported in full (a region of a stale page).
155 pub full_copy: bool,
156 /// Possibly superseded.
157 pub superseded: bool,
158}
159
160impl RegionInfo {
161 /// The bytes its region's data in the stream holds: a slot's object's
162 /// length, an append region's durable length when it has one, else its
163 /// data size.
164 #[must_use]
165 pub fn stream_len(&self) -> u64 {
166 match self.kind {
167 RegionKind::Slot => self.object_len.unwrap_or(0),
168 _ => self.durable_len.unwrap_or(self.data_size),
169 }
170 }
171
172 /// Whether a restore into a store of blocks of `block_size` can take
173 /// it as it says, checked as untrusted input: a valid name, a non-zero
174 /// generation, the block size the target's (a power of two from 512 to
175 /// 65,536), a data size and cap in whole blocks with the cap at least
176 /// the data size, a durable length only on an append region not
177 /// exported in full and at most its data size, an object length only
178 /// on a slot and at most what a slot holds, no slot exported in full,
179 /// and its restored footprint (two header blocks and its data) within
180 /// `u64`.
181 ///
182 /// # Errors
183 ///
184 /// `NotWritten(InvalidOptions)` at the first check that fails.
185 pub fn validate(&self, block_size: u64) -> Result<(), Error> {
186 let bad = || not_written(NotWrittenCause::InvalidOptions, ErrorContext::default());
187 if RegionName::new(&self.name).is_err() || self.generation == 0 {
188 return Err(bad());
189 }
190 let block = self.block_size;
191 if block != block_size || !block.is_power_of_two() || !(512..=1 << 16).contains(&block) {
192 return Err(bad());
193 }
194 if self.data_size % block != 0 {
195 return Err(bad());
196 }
197 if let Some(cap) = self.cap {
198 if cap < self.data_size || cap % block != 0 {
199 return Err(bad());
200 }
201 }
202 let slot = self.kind == RegionKind::Slot;
203 if let Some(d) = self.durable_len {
204 if self.kind != RegionKind::Append || self.full_copy || d > self.data_size {
205 return Err(bad());
206 }
207 }
208 if let Some(n) = self.object_len {
209 let most = max_object(block.trailing_zeros() as u8) as u64;
210 if !slot || n > most {
211 return Err(bad());
212 }
213 }
214 if self.full_copy && slot {
215 return Err(bad());
216 }
217 let footprint = block
218 .checked_mul(2)
219 .and_then(|h| h.checked_add(self.cap.unwrap_or(0).max(self.data_size)));
220 if footprint.is_none() {
221 return Err(bad());
222 }
223 Ok(())
224 }
225
226 /// The format's record of it (`None` if its name is not a region name
227 /// or its object length does not fit the record).
228 pub(crate) fn to_record(&self) -> Option<RegionRecord> {
229 Some(RegionRecord {
230 name: RegionName::new(&self.name).ok()?,
231 kind: self.kind,
232 fill: self.fill,
233 fill_key: self.fill_key,
234 tag: self.tag,
235 generation: self.generation,
236 cap: self.cap,
237 data_size: self.data_size,
238 durable_len: self.durable_len,
239 object_len: match self.object_len {
240 Some(n) => Some(u32::try_from(n).ok()?),
241 None => None,
242 },
243 stream_len: self.stream_len(),
244 full_copy: self.full_copy,
245 superseded: self.superseded,
246 })
247 }
248
249 /// The region a record describes, in a stream of blocks of
250 /// `block_size`.
251 pub(crate) fn from_record(rec: &RegionRecord, block_size: u64) -> Self {
252 Self {
253 name: rec.name.as_str().to_owned(),
254 kind: rec.kind,
255 generation: rec.generation,
256 tag: rec.tag,
257 cap: rec.cap,
258 fill: rec.fill,
259 fill_key: rec.fill_key,
260 block_size,
261 data_size: rec.data_size,
262 durable_len: rec.durable_len,
263 object_len: rec.object_len.map(u64::from),
264 full_copy: rec.full_copy,
265 superseded: rec.superseded,
266 }
267 }
268
269 /// Its bytes in format `exportinfo`'s region record ([`RECORD_LEN`],
270 /// versioned with the record), for a caller's own framing; the block
271 /// size travels in the header ([`ExportHeader::to_bytes`]).
272 ///
273 /// # Errors
274 ///
275 /// `NotWritten(InvalidOptions)` if its fields cannot be a record (a
276 /// name not a region name, an object length past `u32`, fields that
277 /// disagree with its kind).
278 pub fn to_bytes(&self) -> Result<[u8; RECORD_LEN], Error> {
279 let bad = || not_written(NotWrittenCause::InvalidOptions, ErrorContext::default());
280 let rec = self.to_record().ok_or_else(bad)?;
281 rec.check().map_err(|_| bad())?;
282 Ok(rec.encode())
283 }
284
285 /// A region from bytes [`Self::to_bytes`] wrote, in a stream of blocks
286 /// of `block_size`, decoded and then [`Self::validate`]d (the bytes are
287 /// untrusted).
288 ///
289 /// # Errors
290 ///
291 /// `NotWritten(InvalidOptions)` if the bytes are not a record or the
292 /// region fails [`Self::validate`].
293 pub fn from_bytes(bytes: &[u8], block_size: u64) -> Result<Self, Error> {
294 let rec = RegionRecord::decode(bytes)
295 .map_err(|_| not_written(NotWrittenCause::InvalidOptions, ErrorContext::default()))?;
296 let info = Self::from_record(&rec, block_size);
297 info.validate(block_size)?;
298 Ok(info)
299 }
300}
301
302impl ExportHeader {
303 /// Its bytes: format `exportinfo`'s header and lists, sealed by a
304 /// CRC-32C, for a caller's own framing (the regions follow as
305 /// [`RegionInfo::to_bytes`] records).
306 ///
307 /// # Errors
308 ///
309 /// `NotWritten(InvalidOptions)` if it cannot be encoded: a block size
310 /// out of range, a verified name not a region name, a list holding an
311 /// item twice, or a count past `u32`.
312 pub fn to_bytes(&self) -> Result<Vec<u8>, Error> {
313 let bad = || not_written(NotWrittenCause::InvalidOptions, ErrorContext::default());
314 // A power of two (its range the codec's to check).
315 let block = self.block_size;
316 if !block.is_power_of_two() {
317 return Err(bad());
318 }
319 let mut omitted: Vec<u64> = self.facts.omitted.iter().map(|id| id.get()).collect();
320 omitted.sort_unstable();
321 let mut lost = self.facts.lost_pages.clone();
322 lost.sort_unstable();
323 let mut stale = self.facts.stale_pages.clone();
324 stale.sort_unstable();
325 let mut verified = Vec::with_capacity(self.facts.verified_up_to.len());
326 for (name, len) in &self.facts.verified_up_to {
327 verified.push((RegionName::new(name).map_err(|_| bad())?, *len));
328 }
329 verified.sort_by_key(|(n, _)| n.as_padded());
330 let h = InfoHeader {
331 writable: self.writable,
332 block_log2: block.trailing_zeros() as u8,
333 region_count: self.region_count,
334 lend: u32::try_from(self.lend).unwrap_or(u32::MAX),
335 total: self.total,
336 omitted_count: 0,
337 lost_count: 0,
338 stale_count: 0,
339 verified_count: 0,
340 };
341 let counted = InfoHeader {
342 omitted_count: u32::try_from(omitted.len()).map_err(|_| bad())?,
343 lost_count: u32::try_from(lost.len()).map_err(|_| bad())?,
344 stale_count: u32::try_from(stale.len()).map_err(|_| bad())?,
345 verified_count: u32::try_from(verified.len()).map_err(|_| bad())?,
346 ..h
347 };
348 let len = exportinfo::head_len(&counted)
349 .and_then(|n| n.checked_add(4))
350 .ok_or_else(bad)?;
351 let mut out = vec![0u8; len];
352 let n = exportinfo::encode_head(&h, &omitted, &lost, &stale, &verified, &mut out)
353 .map_err(|_| bad())?;
354 out.truncate(n);
355 Ok(out)
356 }
357
358 /// A header from bytes [`Self::to_bytes`] wrote (or a `regions.info`'s
359 /// header and lists, sealed), decoded and checked: the CRC-32C, the
360 /// block size in range, each list in its order. Its `superseded` facts
361 /// are not carried (each region's record says it).
362 ///
363 /// # Errors
364 ///
365 /// `NotWritten(InvalidOptions)` if the bytes are not a sealed header.
366 pub fn from_bytes(bytes: &[u8]) -> Result<Self, Error> {
367 let info = exportinfo::decode_head(bytes)
368 .map_err(|_| not_written(NotWrittenCause::InvalidOptions, ErrorContext::default()))?;
369 Ok(Self::from_view(&info))
370 }
371
372 /// The header a decoded view holds (its lists the volume's facts).
373 pub(crate) fn from_view(info: &exportinfo::ExportInfo<'_>) -> Self {
374 let h = info.header;
375 Self {
376 region_count: h.region_count,
377 block_size: 1u64 << h.block_log2,
378 facts: VolumeFacts {
379 omitted: info.omitted().map(RegionId::from_raw).collect(),
380 lost_pages: info.lost_pages().collect(),
381 stale_pages: info.stale_pages().collect(),
382 verified_up_to: info
383 .verified()
384 .map(|(n, len)| (n.as_str().to_owned(), len))
385 .collect(),
386 superseded: Vec::new(),
387 },
388 writable: h.writable,
389 lend: h.lend as usize,
390 total: h.total,
391 }
392 }
393}
394
395/// Where an import pulls a store from: the header an export's sink was
396/// given, then each region's info and its bytes, in order. A caller's own
397/// source (its framing carried by [`ExportHeader::to_bytes`] and
398/// [`RegionInfo::to_bytes`]) or a [`FileSource`](crate::FileSource).
399/// Everything it yields is untrusted: the import checks it.
400pub trait ImportSource {
401 /// The export's header (first, once).
402 ///
403 /// # Errors
404 ///
405 /// The source's: the import stops, nothing renamed.
406 fn header(&mut self) -> Result<ExportHeader, SinkError>;
407
408 /// The next region's info, or `None` after the last.
409 ///
410 /// # Errors
411 ///
412 /// The source's: the import stops, nothing renamed.
413 fn next_region(&mut self) -> Result<Option<RegionInfo>, SinkError>;
414
415 /// The current region's next bytes into `buf` (at most `buf.len()`):
416 /// how many, 0 at the region's end.
417 ///
418 /// # Errors
419 ///
420 /// The source's: the import stops, nothing renamed.
421 fn read(&mut self, buf: &mut [u8]) -> Result<usize, SinkError>;
422}
423
424/// How far an export got.
425#[derive(Debug, Clone, Default, PartialEq, Eq)]
426#[non_exhaustive]
427pub struct ExportReport {
428 /// Regions exported: each counted once its `end_region` returned `Ok`.
429 pub regions: usize,
430 /// Bytes the sink's `data` accepted.
431 pub bytes: u64,
432 /// The region being exported when the export stopped (`None` once
433 /// every region ended, or before the first).
434 pub region: Option<String>,
435 /// The offset in that region's stream the sink accepted bytes up to.
436 pub offset: u64,
437 /// Whether the store was open for writing.
438 pub writable: bool,
439 /// Every region ended and the sink's `finish` returned `Ok`.
440 pub complete: bool,
441 /// The volume's facts (empty if the plan refused).
442 pub facts: VolumeFacts,
443}
444
445/// A sink's refusal, without payload: an I/O error, a refusal, or the
446/// caller's own code.
447#[derive(Debug, Clone, Copy, PartialEq, Eq)]
448#[non_exhaustive]
449pub enum SinkError {
450 /// An I/O error, as the operating system reported it.
451 Io(OsError),
452 /// The sink refused the call (out of order, or past what it was told).
453 Refused,
454 /// The caller's own code.
455 Other(u32),
456}
457
458impl core::fmt::Display for SinkError {
459 fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
460 match self {
461 Self::Io(raw) => write!(f, "sink I/O error {}", raw.code),
462 Self::Refused => f.write_str("sink refused the call"),
463 Self::Other(code) => write!(f, "sink error {code}"),
464 }
465 }
466}
467
468impl std::error::Error for SinkError {}
469
470/// An export that stopped: why, the sink's own error if it was the sink's
471/// (`error` is then `Io` for [`SinkError::Io`], `NotWritten(Interrupted)`
472/// otherwise), and how far it got (`complete` false).
473#[derive(Debug, Clone)]
474#[non_exhaustive]
475pub struct ExportError {
476 /// Why it stopped (the region in its context when one was being
477 /// exported).
478 pub error: Error,
479 /// The sink's error, if the sink stopped it.
480 pub sink: Option<SinkError>,
481 /// How far it got.
482 pub report: ExportReport,
483}
484
485impl core::fmt::Display for ExportError {
486 fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
487 write!(f, "export stopped: {}", self.error)
488 }
489}
490
491impl std::error::Error for ExportError {
492 fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
493 Some(&self.error)
494 }
495}
496
497/// Where an export pushes a store: its header, then each region's info,
498/// its bytes in order and its end, then the finish. Pacing is invisible to
499/// it: the export's reads are paced by its own call.
500pub trait ExportSink {
501 /// Before the first region.
502 ///
503 /// # Errors
504 ///
505 /// The sink's: the export stops.
506 fn begin(&mut self, header: &ExportHeader) -> Result<(), SinkError>;
507
508 /// Before a region's bytes.
509 ///
510 /// # Errors
511 ///
512 /// The sink's: the export stops.
513 fn region(&mut self, info: &RegionInfo) -> Result<(), SinkError>;
514
515 /// The region's next bytes, contiguous from offset 0 of its stream
516 /// ([`Chunk::offset`]). The sink owns the chunk: it may keep up to the
517 /// header's `lend - 1` and drop them later.
518 ///
519 /// # Errors
520 ///
521 /// The sink's: the export stops; the report's offset is where this
522 /// chunk began.
523 fn data(&mut self, chunk: Chunk) -> Result<(), SinkError>;
524
525 /// Called when the export needs a buffer and the sink holds every one
526 /// lent: drop the chunks no longer needed. If it still holds them all
527 /// the export fails `TooManyInFlight`. By default, nothing.
528 ///
529 /// # Errors
530 ///
531 /// The sink's: the export stops.
532 fn settle(&mut self) -> Result<(), SinkError> {
533 Ok(())
534 }
535
536 /// After a region's last bytes: the region counts as exported once
537 /// this returns `Ok`.
538 ///
539 /// # Errors
540 ///
541 /// The sink's: the export stops.
542 fn end_region(&mut self) -> Result<(), SinkError>;
543
544 /// After the last region (right after `begin` with none): the export is
545 /// complete once this returns `Ok`.
546 ///
547 /// # Errors
548 ///
549 /// The sink's: the export stops, incomplete.
550 fn finish(&mut self) -> Result<(), SinkError>;
551}
552
553/// Why an export stopped, before the report is attached.
554struct Stop {
555 error: Error,
556 sink: Option<SinkError>,
557}
558
559impl From<Error> for Stop {
560 fn from(error: Error) -> Self {
561 Self { error, sink: None }
562 }
563}
564
565/// A refusal naming `region`.
566fn refused(cause: NotWrittenCause, region: Option<RegionId>) -> Error {
567 Error::NotWritten {
568 cause,
569 ctx: ErrorContext {
570 region,
571 ..ErrorContext::default()
572 },
573 }
574}
575
576/// The sink's error as the export's, naming `region`.
577fn sunk(region: Option<RegionId>) -> impl Fn(SinkError) -> Stop {
578 move |e| {
579 let ctx = ErrorContext {
580 region,
581 ..ErrorContext::default()
582 };
583 let error = match e {
584 SinkError::Io(raw) => Error::Io {
585 op: Op::Write,
586 raw,
587 ctx,
588 },
589 SinkError::Refused | SinkError::Other(_) => Error::NotWritten {
590 cause: NotWrittenCause::Interrupted,
591 ctx,
592 },
593 };
594 Stop {
595 error,
596 sink: Some(e),
597 }
598 }
599}
600
601fn too_many(e: &Error) -> bool {
602 matches!(
603 e,
604 Error::NotWritten {
605 cause: NotWrittenCause::TooManyInFlight,
606 ..
607 }
608 )
609}
610
611impl<P: Platform> Store<P>
612where
613 P::Queue: Send,
614{
615 /// Exports every region to `sink`, in the store's order: see the
616 /// module docs and [`Self::export_with`].
617 ///
618 /// # Errors
619 ///
620 /// As [`Self::export_with`].
621 pub fn export(&self, sink: &mut dyn ExportSink) -> Result<ExportReport, Box<ExportError>> {
622 self.export_with(sink, ExportOptions::default())
623 }
624
625 /// Exports the regions `opts` names (all by default) to `sink`: its
626 /// `begin` with the header, then for each region its `region`, its
627 /// bytes through `data` and its `end_region`, then its `finish`.
628 ///
629 /// # Errors
630 ///
631 /// An [`ExportError`] (boxed: it holds the report) with how far it got. Before the sink is
632 /// called: `NotWritten(InvalidOptions)` for a region named twice, a
633 /// name no region has, a depth out of range, or on a writable open a
634 /// region that only a read-only open exports (one of a stale page,
635 /// possibly superseded, or at risk: open the store read-only to export
636 /// it); `NotWritten(NotReady)` for an unavailable region or an entry
637 /// that does not decode, `NotWritten(Frozen)` for a stale or lost table
638 /// page (both unless [`ExportOptions::accept_stale_pages`]);
639 /// `NotWritten(TooLarge)` for a slot object a format-2 slot cannot
640 /// hold; `NotWritten(StaleGeneration)` for a region that changed while
641 /// the plan read it; `NotWritten(PoolExhausted)` with too few buffers
642 /// free for the stream; `Io` if its queue could not be made. At a
643 /// region's turn: `NotWritten(StaleGeneration)` (recycled) or
644 /// `NotWritten(NotReady)` (released) since the plan or while it
645 /// streams, `NotWritten(TooLarge)` for a slot object committed since
646 /// that a format-2 slot cannot hold; a read's errors as a
647 /// [`ReadStream`]'s, a media error (`Corruption(MediaError)`, the
648 /// range named) among them; `NotWritten(TooManyInFlight)` if the sink
649 /// still holds every lent chunk after `settle`. Any error of the sink
650 /// (in `sink`).
651 pub fn export_with(
652 &self,
653 sink: &mut dyn ExportSink,
654 opts: ExportOptions,
655 ) -> Result<ExportReport, Box<ExportError>> {
656 let mut report = ExportReport {
657 writable: !self.inner.read_only,
658 ..ExportReport::default()
659 };
660 match self.export_run(sink, &opts, &mut report) {
661 Ok(()) => {
662 report.region = None;
663 report.offset = 0;
664 report.complete = true;
665 Ok(report)
666 }
667 Err(Stop { error, sink }) => Err(Box::new(ExportError {
668 error,
669 sink,
670 report,
671 })),
672 }
673 }
674
675 /// The export's plan: the options checked, `copy_store`'s plan of the
676 /// regions named, and on a writable open the regions only a read-only
677 /// open exports refused.
678 fn export_plan(&self, opts: &ExportOptions) -> Result<CopyPlan, Error> {
679 // A name no region has is refused by the plan; the depth by the
680 // stream, before the sink's `begin`.
681 if let Some(names) = &opts.regions {
682 if (1..names.len()).any(|i| names[..i].contains(&names[i])) {
683 return Err(not_written(
684 NotWrittenCause::InvalidOptions,
685 ErrorContext::default(),
686 ));
687 }
688 }
689 let plan = self.plan_of(opts.accept_stale_pages, opts.regions.as_deref())?;
690 // A region of a stale page is read by container offset: on a
691 // writable open an accepted rollback may free its extra extent
692 // under it. A superseded or at-risk region may be made writable
693 // and stream newer bytes. Only a read-only open exports them.
694 if !self.inner.read_only {
695 if let Some(f) = plan.facts.iter().find(|f| f.full.is_some() || f.superseded) {
696 return Err(refused(NotWrittenCause::InvalidOptions, Some(f.source.id)));
697 }
698 }
699 Ok(plan)
700 }
701
702 /// An upper bound on what a restore of `facts` takes in a new store of
703 /// the default geometry with blocks of `block` (saturating).
704 fn export_total(facts: &[Facts], block: u64) -> u64 {
705 let defaults = StoreOptions::default();
706 let log2_block = block.trailing_zeros() as u8;
707 let log2_table = defaults.log2_table_slot.clamp(log2_block, 16);
708 let count = u32::try_from(facts.len()).unwrap_or(u32::MAX);
709 let table =
710 crate::v2::layout_for(log2_block, log2_table, defaults.region_capacity.max(count))
711 .data_start();
712 facts.iter().fold(table, |total, f| {
713 let data = match f.kind {
714 RegionKind::Slot => max_object(log2_block) as u64,
715 _ => f.cap.unwrap_or(0).max(f.data_size),
716 };
717 total.saturating_add(2 * block).saturating_add(data)
718 })
719 }
720
721 fn export_run(
722 &self,
723 sink: &mut dyn ExportSink,
724 opts: &ExportOptions,
725 report: &mut ExportReport,
726 ) -> Result<(), Stop> {
727 let plan = self.export_plan(opts)?;
728 // The call's record on this store's device group: the stream's
729 // reads are its pieces; a reference sink joins it at `begin`.
730 let pace = self.inner.domain.new_pace();
731 let call = CallState::new(Arc::clone(&self.inner.qos), pace.clone());
732 let _bound = call::bind(&call);
733 let ran = self.export_body(sink, opts, report, &plan, pace, &call);
734 // A part's failure (a reference sink's write, even one a caller's
735 // wrapper swallowed) is the export's: its error, and the report
736 // clamped to where the failed write began.
737 let failure = call.failure();
738 // To where the failure was in the export's own order of regions:
739 // the regions ended before it, the region and offset when inside
740 // one, and the bytes up to there.
741 let clamp = |report: &mut ExportReport| {
742 if let Some(f) = &failure {
743 // The regions ended before it (a backstop beside the count
744 // of regions every bound sink ended, below: a part that
745 // records a failure without binding as a sink).
746 report.regions = report.regions.min(f.region as usize);
747 let base = call.base(f.region);
748 match plan.facts.get(f.region as usize).filter(|_| f.inside) {
749 Some(r) => {
750 // Never past the bytes `data` accepted when the
751 // export stopped in the failure's region (a write
752 // that failed inside a later chunk's call began
753 // past that chunk's start); a region the export
754 // went past was accepted whole.
755 let same = report.region.as_deref() == Some(r.name.as_str());
756 report.region = Some(r.name.clone());
757 report.offset = if same {
758 report.offset.min(f.offset)
759 } else {
760 f.offset
761 };
762 }
763 None => {
764 report.region = None;
765 report.offset = 0;
766 if let Some(b) = base {
767 report.bytes = report.bytes.min(b);
768 }
769 }
770 }
771 }
772 // Stopped inside a region, whatever stopped it: no further than
773 // every bound reference sink's writes done (a sink holding
774 // accepted bytes not yet written counts none of them), and the
775 // bytes no further than that offset.
776 let inside = match &failure {
777 Some(f) => f.inside.then_some(f.region),
778 None => call.inside_region(),
779 };
780 if let (Some(i), Some(_)) = (inside, report.region.as_ref()) {
781 if let Some(w) = call.written_up_to(i) {
782 report.offset = report.offset.min(w);
783 }
784 if let Some(b) = call.base(i) {
785 report.bytes = report.bytes.min(b.saturating_add(report.offset));
786 }
787 }
788 // A region counts only once every bound reference sink ended
789 // it (a wrapper may have swallowed a sink's failure no part
790 // recorded).
791 if let Some(n) = call.regions_ended() {
792 // Short of where the report stands (the region it names,
793 // else past the regions it counts): it stands at region
794 // `n` instead, the first a sink never ended, with that
795 // region's writes done, and nothing past it.
796 let i = n as usize;
797 let named = report
798 .region
799 .as_ref()
800 .and_then(|name| plan.facts.iter().position(|f| &f.name == name));
801 if i < named.unwrap_or(report.regions) {
802 report.regions = i;
803 report.region = plan.facts.get(i).map(|f| f.name.clone());
804 report.offset = call.written_up_to(n).unwrap_or(0);
805 if let Some(b) = call.base(n) {
806 report.bytes = report.bytes.min(b.saturating_add(report.offset));
807 }
808 }
809 }
810 };
811 match ran {
812 Err(mut stop) => {
813 if let Some(f) = &failure {
814 stop.error = f.error.clone();
815 }
816 clamp(report);
817 Err(stop)
818 }
819 Ok(()) => {
820 if let Some(f) = &failure {
821 clamp(report);
822 return Err(Stop {
823 error: f.error.clone(),
824 sink: None,
825 });
826 }
827 // A bound reference sink that never finished (a wrapper
828 // that dropped it, or fell back to another).
829 if !call.all_finished() {
830 clamp(report);
831 return Err(Stop::from(not_written(
832 NotWrittenCause::Interrupted,
833 ErrorContext::default(),
834 )));
835 }
836 Ok(())
837 }
838 }
839 }
840
841 /// The export's sink calls, bound to its call.
842 fn export_body(
843 &self,
844 sink: &mut dyn ExportSink,
845 opts: &ExportOptions,
846 report: &mut ExportReport,
847 plan: &CopyPlan,
848 pace: Option<crate::pace::Pace>,
849 call: &CallState,
850 ) -> Result<(), Stop> {
851 report.facts = VolumeFacts {
852 omitted: plan.omitted.clone(),
853 lost_pages: plan.lost_pages.clone(),
854 stale_pages: plan.stale_pages.clone(),
855 verified_up_to: plan.verified_up_to.clone(),
856 superseded: plan.superseded.clone(),
857 };
858 let stream = self.export_stream(opts.depth, pace)?;
859 let header = ExportHeader {
860 region_count: u32::try_from(plan.facts.len()).unwrap_or(u32::MAX),
861 block_size: plan.block,
862 facts: report.facts.clone(),
863 writable: report.writable,
864 lend: stream.lend(),
865 total: Self::export_total(&plan.facts, plan.block),
866 };
867 sink.begin(&header).map_err(sunk(None))?;
868 // A sink that joined the source's record shares its pieces.
869 if call.shares_source() {
870 stream.make_second();
871 }
872 for (i, f) in plan.facts.iter().enumerate() {
873 report.region = Some(f.name.clone());
874 report.offset = 0;
875 let index = u32::try_from(i).unwrap_or(u32::MAX);
876 call.enter(index, report.bytes);
877 self.export_region(f, plan.block, &stream, sink, report)
878 .map_err(|stop| Self::part_error(&stream, stop))?;
879 report.regions += 1;
880 call.leave(index.saturating_add(1));
881 }
882 report.region = None;
883 report.offset = 0;
884 // Every part asked before the export reports anything: a read the
885 // stream lost while a sink's wait collected it is its failure.
886 if let Some(e) = stream.failure() {
887 return Err(Stop::from(e));
888 }
889 // Nothing is in flight: every part was read to its end.
890 stream.cancel()?;
891 sink.finish().map_err(sunk(None))
892 }
893
894 /// The stream's own failure, if a collection for another part of the
895 /// call lost its reads (the sink then only saw its wait end), else
896 /// `stop` as it was.
897 fn part_error(stream: &ReadStream<P>, stop: Stop) -> Stop {
898 match stream.failure() {
899 Some(error) => Stop {
900 error,
901 sink: stop.sink,
902 },
903 None => stop,
904 }
905 }
906
907 /// Slot `r`'s committed object at its turn, read under its header lock
908 /// (a slot is never recycled or released: its gate stays live), refused
909 /// as the plan refuses one a format-2 slot cannot hold.
910 fn object_at_turn(&self, r: &LiveRegion, block: u64) -> Result<Option<Vec<u8>>, Error> {
911 let header = r
912 .header
913 .lock()
914 .unwrap_or_else(std::sync::PoisonError::into_inner)
915 .clone();
916 let object = committed_object(&header, &self.inner.layout);
917 if object
918 .as_ref()
919 .is_some_and(|o| o.len() > max_object(block.trailing_zeros() as u8))
920 {
921 return Err(refused(NotWrittenCause::TooLarge, Some(r.id)));
922 }
923 Ok(object)
924 }
925
926 /// Exports region `f` (of the plan) at its turn.
927 fn export_region(
928 &self,
929 f: &Facts,
930 block: u64,
931 stream: &ReadStream<P>,
932 sink: &mut dyn ExportSink,
933 report: &mut ExportReport,
934 ) -> Result<(), Stop> {
935 // Still the planned (id, generation): a recycle or release closes
936 // the planned region's gate for good (a recycle's new generation is
937 // a region of its own), so a gate still live is the plan's region.
938 let r = &f.source;
939 r.gate.live().map_err(|why| retired_error(why, r))?;
940 let id = Some(r.id);
941 let (durable_len, object, part) = match (f.kind, &f.full) {
942 (RegionKind::Slot, _) => (None, self.object_at_turn(r, block)?, None),
943 // By container offset: its capacity's runs, then its extra
944 // extent.
945 (_, Some((ranges, _))) => (
946 None,
947 None,
948 Some((f.data_size, Locator::Ranges(ranges.clone()))),
949 ),
950 (kind, None) => {
951 // A positioned append region's durable prefix, clamped to
952 // its planned data size (still a valid prefix).
953 let durable = (kind == RegionKind::Append && r.positioned.load(Ordering::Acquire))
954 .then(|| {
955 r.frontier
956 .as_ref()
957 .map_or(0, crate::frontier::AppendFrontier::durable)
958 .min(f.data_size)
959 });
960 (
961 durable,
962 None,
963 Some((durable.unwrap_or(f.data_size), Locator::Map)),
964 )
965 }
966 };
967 let info = RegionInfo {
968 name: f.name.clone(),
969 kind: f.kind,
970 generation: f.attrs.generation,
971 tag: f.attrs.tag,
972 cap: f.cap,
973 fill: f.attrs.fill,
974 fill_key: f.attrs.fill_key,
975 block_size: block,
976 data_size: f.data_size,
977 durable_len,
978 object_len: object.as_ref().map(|o| o.len() as u64),
979 full_copy: f.full.is_some(),
980 superseded: f.superseded,
981 };
982 sink.region(&info).map_err(sunk(id))?;
983 if let Some(o) = object.filter(|o| !o.is_empty()) {
984 let mut settled = false;
985 let chunk = loop {
986 match stream.lend_bytes(r, 0, &o) {
987 Ok(c) => break c,
988 Err(e) if too_many(&e) && !settled => {
989 sink.settle().map_err(sunk(id))?;
990 settled = true;
991 }
992 Err(e) => return Err(e.into()),
993 }
994 };
995 sink.data(chunk).map_err(sunk(id))?;
996 report.offset = o.len() as u64;
997 report.bytes += o.len() as u64;
998 }
999 if let Some((len, loc)) = part {
1000 self.export_bytes(r, len, loc, stream, sink, report)?;
1001 }
1002 sink.end_region().map_err(sunk(id))
1003 }
1004
1005 /// Streams `[0, len)` of `r` (read as `loc` places it) to the sink.
1006 fn export_bytes(
1007 &self,
1008 r: &Arc<LiveRegion>,
1009 len: u64,
1010 loc: Locator,
1011 stream: &ReadStream<P>,
1012 sink: &mut dyn ExportSink,
1013 report: &mut ExportReport,
1014 ) -> Result<(), Stop> {
1015 let id = Some(r.id);
1016 stream.add_part(r, 0..len, loc)?;
1017 let mut got = 0u64;
1018 let mut settled = false;
1019 while got < len {
1020 match stream.pull() {
1021 Some(Ok(c)) => {
1022 let n = c.bytes().len() as u64;
1023 // In order from the region's start, with no gap and
1024 // nothing past its end.
1025 if n == 0 || c.offset() != got || n > len - got {
1026 let mut e = crate::store::corrupt();
1027 if let Error::Corruption { ctx, .. } = &mut e {
1028 *ctx = ctx_range(r, got, len);
1029 }
1030 return Err(e.into());
1031 }
1032 settled = false;
1033 sink.data(c).map_err(sunk(id))?;
1034 got += n;
1035 report.offset = got;
1036 report.bytes += n;
1037 }
1038 Some(Err(e)) if too_many(&e) && !settled => {
1039 sink.settle().map_err(sunk(id))?;
1040 settled = true;
1041 }
1042 Some(Err(e)) => return Err(e.into()),
1043 // Ended before the region's bytes did.
1044 None => {
1045 return Err(
1046 not_written(NotWrittenCause::Interrupted, ctx_range(r, got, len)).into(),
1047 );
1048 }
1049 }
1050 }
1051 Ok(())
1052 }
1053}