lsm_tree/repair.rs
1// SPDX-License-Identifier: Apache-2.0
2// Copyright (c) 2026-present, Dmitry Prudnikov
3
4//! Last-resort `MANIFEST` reconstruction from the SST files on disk.
5//!
6//! Once a tree has a `MANIFEST`, that manifest is a single point of failure for
7//! the database as a whole: a corrupt manifest means the tree cannot open at
8//! all, even when every SST on disk is intact. Repair scans the table folder(s),
9//! reads each SST's own metadata, and writes a fresh manifest referencing what
10//! is actually present.
11//!
12//! ## What is recovered, what is lost
13//!
14//! Every readable SST is preserved. What the rebuilt manifest cannot know is the
15//! LSM level structure (which file lived at which level) and any version edits
16//! that had not yet been durably logged (an in-flight compaction's output
17//! placement, recent table deletions). Following the RocksDB `RepairDB()`
18//! pattern, all recovered SSTs are placed at L0 ordered by sequence number
19//! (newest first) and a normal background compaction redistributes them into
20//! proper levels on the next open. Reads are correct throughout: L0 permits
21//! overlapping runs, and the merge reader resolves the latest value by sequence
22//! number regardless of physical placement.
23//!
24//! ## Correctness of the recomputed table checksum
25//!
26//! The manifest binds each table by its whole-file XXH3-128 checksum. A normal
27//! write computes that digest incrementally as the file is streamed out, and the
28//! file is written strictly sequentially (no seek-back rewrites after the digest
29//! is taken), so the on-disk bytes equal the hashed byte stream. Repair therefore
30//! recomputes the identical digest by streaming the file start to end. The data
31//! itself is protected independently by per-block checksums, which
32//! [`Table::recover`] validates as it parses, so an SST that survives recovery is
33//! structurally sound.
34//!
35//! ## Scope
36//!
37//! KV-separated (blob) trees are supported: the `blobs/` folder is scanned to
38//! rediscover the blob files and record them in the rebuilt manifest. Blob-file
39//! fragmentation statistics cannot be reconstructed from a directory scan
40//! (they are derived from compaction history), so they start empty; blob GC is
41//! advisory and re-learns reclaimable space over time without dropping data.
42
43use crate::{
44 SeqNo, Table, TableId, UserKey,
45 config::{Config, TreeType},
46 version::{BlobFileList, Level, Run, Version},
47};
48use std::{path::PathBuf, sync::Arc};
49
50/// Per-file repair failures: `(path, human-readable reason)`. Mirrors
51/// [`RepairReport::unreadable_files`].
52type UnreadableFiles = Vec<(PathBuf, String)>;
53
54/// Outcome of a [`Config::repair`] run.
55///
56/// `recovered` plus `unreadable` accounts for every SST-named file the scan
57/// considered. `unreadable_files` carries the per-file reason a file was skipped
58/// so an operator can decide whether to investigate or discard it.
59#[derive(Debug)]
60pub struct RepairReport {
61 /// Number of SSTs whose metadata parsed and that are now referenced by the
62 /// rebuilt manifest (including any recovered by salvage; see [`salvaged`]).
63 ///
64 /// [`salvaged`]: RepairReport::salvaged
65 pub recovered: usize,
66
67 /// Of [`recovered`](RepairReport::recovered), how many were recovered by
68 /// block-level salvage (their original failed whole-file recovery, so the
69 /// salvaged copy may be missing the key ranges of corrupt blocks). Always
70 /// zero unless repair ran with salvage enabled
71 /// ([`Config::repair_with_salvage`]).
72 pub salvaged: usize,
73
74 /// Number of SST-named files that could not be opened or parsed and were
75 /// therefore left out of the manifest.
76 pub unreadable: usize,
77
78 /// Path and human-readable error for each unreadable file.
79 pub unreadable_files: Vec<(PathBuf, String)>,
80
81 /// HEALTHY files the rebuild deliberately left out because their content
82 /// lives on in the kept tables: a derived compaction output whose inputs
83 /// all survived, or an input a surviving output (chain) fully covers.
84 /// These opened and verified fine — they are not failures, contribute no
85 /// lost coverage, and are removed once the rebuilt manifest is durable.
86 /// `(path, reason)` per exclusion.
87 pub excluded_files: Vec<(PathBuf, String)>,
88
89 /// Key coverage the rebuilt manifest LOST, for every excluded table whose
90 /// metadata still parsed: `(path, first key, last key, highest seqno)`.
91 ///
92 /// Losing a table's bytes loses what it said about those keys. Older
93 /// versions of them survive in other tables and become visible again — a
94 /// value the lost table had overwritten, or a key its tombstone had
95 /// deleted. No repair can tell those apart without the lost bytes, and
96 /// deleting the range instead would destroy intact data (see
97 /// [`repair_with_resurrection`](Config::repair_with_resurrection)), so the
98 /// range is reported rather than acted on: within these bounds, at or below
99 /// the given sequence number, the tree may now serve a superseded value.
100 ///
101 /// The sequence bound is `None` when the table's own sequence base lived in
102 /// the lost manifest — a bulk-ingested SST stores every entry at local
103 /// seqno `0` and takes its effective ordering from a manifest-only offset,
104 /// which is exactly why such a table is excluded. Reporting the on-disk
105 /// local maximum there would scope the affected history far too low, so the
106 /// bound is reported as unknown and the whole history of that key range has
107 /// to be treated as affected.
108 ///
109 /// A KEPT salvaged copy contributes an entry too: it dropped corrupt
110 /// blocks (or blob records), so within its bounds a superseded value may
111 /// likewise now be served, even though the table itself is in the
112 /// manifest. The entry carries the SOURCE's coverage (captured while its
113 /// metadata was readable), not the replacement's: salvage may have
114 /// dropped the block holding the source's outermost keys or highest
115 /// seqno, and bounds derived from the survivors would exclude exactly
116 /// the lost part.
117 ///
118 /// Empty when nothing was lost. A table whose metadata was unreadable
119 /// contributes no entry here — its coverage is unknowable, and it is
120 /// listed in [`unknowable_losses`](Self::unknowable_losses) instead.
121 pub lost_coverage: Vec<(PathBuf, UserKey, UserKey, Option<SeqNo>)>,
122
123 /// Table files whose loss cannot be scoped at all: their metadata never
124 /// parsed, so neither the affected key range nor a seqno bound is
125 /// derivable. Covers both an EXCLUDED unreadable table and a KEPT lossy
126 /// salvaged copy whose source's metadata was unreadable (the copy's own
127 /// metadata only bounds what survived). Any entry here forces
128 /// [`wal_replay_scope`](Self::wal_replay_scope) to
129 /// [`WalReplayScope::FullHistory`], since no bound can prove a retained
130 /// record is NOT affected. Blob files are not listed: losing blob content
131 /// surfaces through the referencing tables, which the other fields cover.
132 pub unknowable_losses: Vec<PathBuf>,
133
134 /// Damaged blob files whose salvaged replacement IS installed in the
135 /// rebuilt manifest: the canonical (installed) path plus a note on what
136 /// was recovered and where the damaged original is preserved. Disjoint
137 /// from [`unreadable_files`](RepairReport::unreadable_files), which lists
138 /// only files LEFT OUT of the manifest. Empty for standard trees and for
139 /// repairs where no blob file needed salvage.
140 pub blob_files_salvaged: Vec<(PathBuf, String)>,
141
142 /// Dictionary files whose bytes no longer hashed to their name, moved to
143 /// `{id}.damaged` beside them once the rebuilt manifest was durable:
144 /// `(new path, reason)` each. Left under its name, such a file would fail
145 /// every open of the repaired tree. A table that needed one is not in this
146 /// list: it fails the repair instead, which a re-run supplying that
147 /// dictionary recovers from.
148 pub damaged_dictionaries: Vec<(PathBuf, String)>,
149
150 /// Description of the level-assignment strategy used (constant for now;
151 /// surfaced so the report is self-explanatory and forward-compatible).
152 pub method: &'static str,
153
154 /// Operator-facing caveats about the rebuilt state.
155 pub warnings: Vec<&'static str>,
156
157 /// Identifies this repair's report. The tree hands a report out until
158 /// [`AbstractTree::acknowledge_repair_report`](crate::AbstractTree::acknowledge_repair_report)
159 /// takes this id, so it may arrive more than once; a caller that records
160 /// the ids it applied skips a repeat.
161 pub id: u128,
162
163 /// The ids of earlier reports this one takes in: they were still owed
164 /// when this repair ran, and their losses are listed here as well. A
165 /// caller that applied one of them already has replayed its losses.
166 pub includes: Vec<u128>,
167}
168
169/// What an external write-ahead log must replay after this repair, derived
170/// from [`RepairReport::lost_coverage`] by
171/// [`RepairReport::wal_replay_scope`].
172///
173/// A repair can REGRESS persisted state below the WAL's trim watermark `W`
174/// (a dropped or salvaged table loses versions at or below `W`, while
175/// [`get_highest_persisted_seqno`] stays high because neighbouring tables
176/// survived), so the standard `seqno > W` tail replay is not always
177/// sufficient. See `docs/external-wal.md` § Replay after repair for the full
178/// recipe, including why a merge operand needs a presence check
179/// ([`scan_since_seqno_in_range`]) while a put / delete may be re-applied
180/// blindly.
181///
182/// [`get_highest_persisted_seqno`]: crate::AbstractTree::get_highest_persisted_seqno
183/// [`scan_since_seqno_in_range`]: crate::Tree::scan_since_seqno_in_range
184#[derive(Debug, Clone, Copy, PartialEq, Eq)]
185pub enum WalReplayScope {
186 /// Nothing was excluded: the standard tail replay (`seqno > W`) is
187 /// sufficient.
188 TailOnly,
189 /// Coverage was lost at or below this sequence number. In addition to the
190 /// tail, replay every RETAINED record whose key falls inside a
191 /// [`RepairReport::lost_coverage`] range and whose seqno is at or below
192 /// this bound (presence-checking merge operands).
193 LostUpTo(SeqNo),
194 /// At least one excluded table's sequence base was itself lost with the
195 /// manifest, so no seqno bound scopes the damage: the whole retained
196 /// history must be reconciled (still presence-checking merge operands).
197 ///
198 /// Read [`RepairReport::unknowable_losses`] to know over WHAT. When it is
199 /// EMPTY the damage is still localized, so the
200 /// [`lost_coverage`](RepairReport::lost_coverage) ranges bound the work.
201 /// When it is NON-EMPTY the loss has no key range either — that table's
202 /// metadata never parsed — and the reconciliation runs over the ENTIRE
203 /// keyspace in ONE pass; iterating the known ranges on top of it would
204 /// subtract the same survivors twice.
205 FullHistory,
206}
207
208impl RepairReport {
209 /// Derives the WAL replay obligation from
210 /// [`lost_coverage`](Self::lost_coverage) and
211 /// [`unknowable_losses`](Self::unknowable_losses):
212 /// [`WalReplayScope::TailOnly`] when nothing was lost,
213 /// [`WalReplayScope::FullHistory`] when any loss is unscopable (an
214 /// unknown seqno bound, or an excluded table whose coverage never
215 /// parsed), else [`WalReplayScope::LostUpTo`] the highest lost bound. The
216 /// per-range detail (which KEYS are affected) stays in `lost_coverage`;
217 /// this is the aggregate a WAL uses to decide how far back its archive
218 /// must reach.
219 #[must_use]
220 pub fn wal_replay_scope(&self) -> WalReplayScope {
221 if !self.unknowable_losses.is_empty() {
222 return WalReplayScope::FullHistory;
223 }
224 let mut ceiling: Option<SeqNo> = None;
225 for (_, _, _, bound) in &self.lost_coverage {
226 match bound {
227 None => return WalReplayScope::FullHistory,
228 Some(b) => ceiling = Some(ceiling.map_or(*b, |c| c.max(*b))),
229 }
230 }
231 match ceiling {
232 None => WalReplayScope::TailOnly,
233 Some(b) => WalReplayScope::LostUpTo(b),
234 }
235 }
236}
237
238/// [`RepairReport::method`] of every repair.
239const REPAIR_METHOD: &str = "all-to-L0 with sequence-number ordering";
240
241const WARN_ALL_AT_L0: &str =
242 "All recovered tables placed at L0; background compaction will redistribute them";
243const WARN_UNLOGGED_EDITS: &str =
244 "Recent unlogged version edits (in-flight compactions, recent deletions) are lost";
245const WARN_BLOB_FRAGMENTATION: &str = "Blob fragmentation stats reset (punched prefixes reseeded); blob GC re-learns the rest over time";
246const WARN_REPORT_COPY_DAMAGED: &str =
247 "The kept report of an earlier repair is damaged; its losses are unscoped";
248
249/// Every warning a report can carry; a kept copy of a report records each by
250/// its index here.
251const WARNINGS: [&str; 4] = [
252 WARN_ALL_AT_L0,
253 WARN_UNLOGGED_EDITS,
254 WARN_BLOB_FRAGMENTATION,
255 WARN_REPORT_COPY_DAMAGED,
256];
257
258/// Adds what `earlier` lost to `report`: a repair over the snapshot of an
259/// earlier repair whose report never reached the caller owes that repair's
260/// replay too. The counts stay `report`'s own, describing this run.
261///
262/// A file left out of the manifest is listed once. A loss is listed once per
263/// distinct entry, not per path: a repair of an earlier repair's salvaged
264/// replacement names the same path with only what the replacement held, and
265/// the earlier, wider range is still owed, as is the note saying where an
266/// earlier salvage kept a damaged original.
267#[cfg(feature = "std")]
268fn fold_report(report: &mut RepairReport, earlier: RepairReport) {
269 fn add<T>(into: &mut Vec<T>, from: Vec<T>, same: impl Fn(&T, &T) -> bool) {
270 for entry in from {
271 if !into.iter().any(|known| same(known, &entry)) {
272 into.push(entry);
273 }
274 }
275 }
276 add(
277 &mut report.unreadable_files,
278 earlier.unreadable_files,
279 |a, b| a.0 == b.0,
280 );
281 report.unreadable = report.unreadable_files.len();
282 add(
283 &mut report.excluded_files,
284 earlier.excluded_files,
285 |a, b| a.0 == b.0,
286 );
287 add(
288 &mut report.lost_coverage,
289 earlier.lost_coverage,
290 PartialEq::eq,
291 );
292 add(
293 &mut report.unknowable_losses,
294 earlier.unknowable_losses,
295 PartialEq::eq,
296 );
297 add(
298 &mut report.blob_files_salvaged,
299 earlier.blob_files_salvaged,
300 PartialEq::eq,
301 );
302 add(
303 &mut report.damaged_dictionaries,
304 earlier.damaged_dictionaries,
305 PartialEq::eq,
306 );
307 for warning in earlier.warnings {
308 if !report.warnings.contains(&warning) {
309 report.warnings.push(warning);
310 }
311 }
312 for id in core::iter::once(earlier.id).chain(earlier.includes) {
313 if !report.includes.contains(&id) {
314 report.includes.push(id);
315 }
316 }
317}
318
319/// Streams `path` from byte `start` to end through XXH3-128. `start == 0`
320/// reproduces the whole-file digest a normal write accumulates; a non-zero
321/// `start` digests only the LIVE suffix of a tight-space RESTRICTED table,
322/// whose `[0, start)` prefix was hole-punched (reads back as zeros) once a
323/// superseding output table took over those keys. The suffix bytes are
324/// untouched by the punch, so this digest is stable across it.
325pub(crate) fn compute_table_checksum_from(
326 fs: &dyn crate::fs::Fs,
327 path: &std::path::Path,
328 start: u64,
329) -> crate::Result<u128> {
330 // The offset-only case is the override-splicing digest with no overrides.
331 compute_table_checksum_with_overrides(fs, path, start, &[])
332}
333
334/// Streams `path` start to end through XXH3-128, matching the digest a normal
335/// table write accumulates via `ChecksummedWriter`.
336pub(crate) fn compute_table_checksum(
337 fs: &dyn crate::fs::Fs,
338 path: &std::path::Path,
339) -> crate::Result<u128> {
340 // The whole-file case is the override-splicing digest from offset 0 with no
341 // overrides: one shared read loop in `compute_table_checksum_with_overrides`.
342 compute_table_checksum_with_overrides(fs, path, 0, &[])
343}
344
345/// As [`compute_table_checksum`], but streams the file with `overrides` spliced
346/// in: each `(offset, bytes)` replaces the on-disk bytes at `[offset,
347/// offset + bytes.len())`. Used to predict the digest an in-place heal WILL
348/// produce, from the corrected block frames it will write, before any write
349/// lands — so the heal attestation can bind that intended post-heal state.
350///
351/// The overrides are size-preserving block frames at distinct, non-overlapping
352/// offsets (the heal rewrites each corrupt block at its existing offset and
353/// size), so splicing them is a byte-for-byte substitution that keeps the file
354/// length and every other byte unchanged. `start` matches
355/// [`compute_table_checksum_from`]: a restricted view predicts only its live
356/// suffix (its corrections all lie there), so the digest starts at the punch
357/// offset.
358pub(crate) fn compute_table_checksum_with_overrides(
359 fs: &dyn crate::fs::Fs,
360 path: &std::path::Path,
361 start: u64,
362 overrides: &[(u64, Vec<u8>)],
363) -> crate::Result<u128> {
364 crate::file::checksum_from_with_overrides(fs, path, start, overrides)
365}
366
367/// Highest id any manifest file in `folder` carries (`v{N}` or `edits-{N}`), if
368/// any. The rebuilt manifest uses `max + 1` so it supersedes every stale
369/// generation and its install never meets a log a predecessor left.
370///
371/// A directory-read failure is propagated (not swallowed as "no versions"): a
372/// transient scan error must not silently reset the version chain to `0` and
373/// risk reusing a live version id.
374fn highest_existing_version_id(
375 fs: &dyn crate::fs::Fs,
376 folder: &std::path::Path,
377) -> crate::Result<Option<u64>> {
378 Ok(fs
379 .read_dir(folder)?
380 .into_iter()
381 .filter_map(|e| {
382 crate::version::recovery::snapshot_id_of(&e.file_name)
383 .or_else(|| crate::version::recovery::log_id_of(&e.file_name))
384 })
385 .max())
386}
387
388/// Removes a file the rebuilt manifest does not name, together with any
389/// companion `.restrict-bound` sidecar, and makes the removal durable.
390///
391/// A repair has exactly two outcomes: a committed tree that opens, or an error.
392/// A file left behind is neither — `Tree::open` rejects a foreign name in
393/// `tables/` and sweeps an id the manifest does not reference, so a removal that
394/// cannot happen here becomes an open that fails. The removal is therefore not
395/// best-effort: a failure fails the repair.
396///
397/// The bytes are NOT preserved anywhere. Recovering the content of a damaged
398/// file is replication, checkpoint plus journal replay, or a backup; a directory
399/// hidden beside the tree recovers nothing and only makes a later run derive
400/// from a different world than the one it was handed.
401///
402/// Callers run this strictly AFTER the manifest commit. Until then every source
403/// is still the only copy of its rows, and a crash must leave the directory
404/// exactly as the retry expects to find it.
405///
406/// `NotFound` counts as done, so a retry finishes a sweep a crash interrupted.
407fn discard_unreferenced(
408 fs: &dyn crate::fs::Fs,
409 path: &std::path::Path,
410 sync_mode: crate::fs::SyncMode,
411) -> crate::Result<()> {
412 let remove = |target: &std::path::Path| -> crate::Result<()> {
413 match fs.remove_file(target) {
414 Ok(()) => Ok(()),
415 Err(e) if e.kind() == crate::io::ErrorKind::NotFound => Ok(()),
416 Err(e) => Err(e.into()),
417 }
418 };
419 // Sidecar first: an SST removed while its `.restrict-bound` survives leaves a
420 // bound that a later run would match by id against an unrelated table. Most
421 // files have none (blob files never do), so probe before asking.
422 let sidecar = crate::restrict_bound::sidecar_path(path);
423 if fs.exists(&sidecar)? {
424 remove(&sidecar)?;
425 }
426 remove(path)?;
427 if let Some(dir) = path.parent() {
428 fs.sync_directory_with(dir, sync_mode)?;
429 }
430 Ok(())
431}
432
433pub(crate) use crate::file::{REPAIR_TMP_SUFFIX, table_id_from_repair_tmp_name};
434
435/// `{id}` -> `{id}.repair-tmp`, the path a repair builds a replacement at.
436///
437/// The replacement CANNOT be built at `{id}` directly: that would either destroy
438/// the source before the manifest commit (a crash then loses the only copy) or,
439/// under a fresh id beside the source, leave both readable after a crash — and
440/// the retry, unable to tell a half-finished replacement from a real table,
441/// would rebuild BOTH into L0, applying every merge operand of that history
442/// twice. A name no scan adopts has neither problem: the source stays intact and
443/// authoritative until the commit, and a leftover temp is garbage.
444fn repair_tmp_path(table_path: &std::path::Path) -> PathBuf {
445 let mut name = table_path.file_name().unwrap_or_default().to_os_string();
446 name.push(REPAIR_TMP_SUFFIX);
447 table_path.with_file_name(name)
448}
449
450/// Swaps a finished replacement onto the name the committed manifest gives it:
451/// `{id}.repair-tmp` becomes `{id}`, destroying the damaged source it replaces,
452/// with any companion `.restrict-bound` sidecar carried along.
453///
454/// Strictly POST-COMMIT. Before the commit the source is still the only copy of
455/// its rows and the manifest still names it; after it, the manifest names this
456/// content and the source is what the tree no longer references.
457///
458/// Sidecar first, and the sidecar's own removal-or-rename settled before the
459/// table moves: a table adopted at `{id}` while a STALE `{id}.restrict-bound`
460/// survives would be reopened restricted at an unrelated bound, silently hiding
461/// its prefix.
462///
463/// `manifest_restricted` is whether the COMMITTED manifest restricts this id.
464/// It disambiguates a retry of an interrupted swap: with the sidecar step
465/// preceding the table rename, a crash between them leaves the replacement's
466/// sidecar already at the destination and no temp sidecar — a state that is
467/// byte-identical to "unrestricted replacement beside the source's stale
468/// sidecar". Only the manifest can tell them apart, and it is the authority
469/// for the bound anyway.
470pub(crate) fn commit_repair_tmp(
471 fs: &dyn crate::fs::Fs,
472 tmp_path: &std::path::Path,
473 table_path: &std::path::Path,
474 sync_mode: crate::fs::SyncMode,
475 manifest_restricted: bool,
476) -> crate::Result<()> {
477 let tmp_sidecar = crate::restrict_bound::sidecar_path(tmp_path);
478 let dest_sidecar = crate::restrict_bound::sidecar_path(table_path);
479 if fs.exists(&tmp_sidecar)? {
480 fs.rename(&tmp_sidecar, &dest_sidecar)?;
481 } else if manifest_restricted {
482 // The manifest restricts this id, so a sidecar already at the
483 // destination is NOT the source's stale metadata — it is the
484 // replacement's own sidecar, moved by an interrupted earlier attempt.
485 // Deleting it would adopt a restricted replacement unrestricted on the
486 // next scan (the fresh copy is unpunched, so no geometry re-derives
487 // the bound) and resurrect the straddling block's sub-bound rows.
488 // Keep it; a missing or disagreeing sidecar is republished from the
489 // manifest by the next open.
490 } else if fs.exists(&dest_sidecar)? {
491 // The replacement is unrestricted, so the source's bound describes bytes
492 // that are about to stop existing. Left in place it would reopen the
493 // replacement restricted at an unrelated bound and hide its prefix.
494 match fs.remove_file(&dest_sidecar) {
495 Ok(()) => {}
496 Err(e) if e.kind() == crate::io::ErrorKind::NotFound => {}
497 Err(e) => return Err(e.into()),
498 }
499 }
500 fs.rename(tmp_path, table_path)?;
501 if let Some(dir) = table_path.parent() {
502 fs.sync_directory_with(dir, sync_mode)?;
503 }
504 Ok(())
505}
506
507/// Decides whether a leftover `{id}.repair-tmp` is the file the committed
508/// manifest describes. The manifest names `id` in BOTH crash cases — after the
509/// commit (the entry describes the replacement) and before it (the entry still
510/// describes the source) — so id membership alone cannot tell a committed swap
511/// from an abandoned build, and swapping the latter in would destroy the file
512/// the manifest actually names (a mid-build crash even leaves the temp
513/// truncated). The entry's checksum can tell them apart: only a committed
514/// repair recorded the temp's digest — the live-suffix digest for a restricted
515/// replacement, whose punch offset comes from the temp's own index. A temp
516/// that does not even open is the mid-build crash; only transient I/O
517/// propagates for a retry.
518///
519/// A temp whose bytes no longer READ BACK at all (a rotted sector, a build
520/// truncated mid-write — non-environmental either way) must not hold the
521/// tree hostage: when the ORIGINAL `{id}` still matches the manifest's
522/// checksum, the manifest provably names the source, so the unreadable temp
523/// is a disposable abandoned build (`Ok(false)`). The temp's failure
524/// propagates only when the original cannot be proven authoritative either —
525/// the temp could then be a committed-but-unswapped replacement, and
526/// discarding it would destroy the only copy the manifest describes.
527#[cfg(feature = "std")]
528pub(crate) fn repair_tmp_is_published(
529 config: &Config,
530 fs: &Arc<dyn crate::fs::Fs>,
531 tmp_path: &std::path::Path,
532 table_id: TableId,
533 manifest_checksum: crate::Checksum,
534 restriction: Option<&crate::UserKey>,
535) -> crate::Result<bool> {
536 // Whether the ORIGINAL `{id}` beside the temp is provably the file the
537 // manifest names — hashed on the SAME basis the manifest recorded it. A
538 // RESTRICTED table's entry holds its LIVE-SUFFIX digest, so hashing the
539 // whole original (its reclaimed prefix included) could never match, and
540 // the temp's ambiguity could never be resolved: every open and every
541 // repair would keep failing on a disposable leftover.
542 let original_is_authoritative = || -> crate::Result<bool> {
543 let original = tmp_path.with_file_name(table_id.to_string());
544 let digest = match restriction {
545 None => compute_table_checksum(&**fs, &original),
546 Some(bound) => {
547 let table = match crate::table::Table::recover(repair_recover_params(
548 config,
549 original.clone(),
550 manifest_checksum,
551 table_id,
552 Arc::clone(fs),
553 None,
554 )) {
555 Ok(table) => table,
556 Err(e) if is_environmental(&e) => return Err(e),
557 // An original that does not open proves nothing.
558 Err(_) => return Ok(false),
559 };
560 match table.punch_offset_for(bound.as_ref()) {
561 Ok(offset) => compute_table_checksum_from(&**fs, &original, offset),
562 Err(e) if is_environmental(&e) => return Err(e),
563 Err(_) => return Ok(false),
564 }
565 }
566 };
567 match digest {
568 Ok(digest) => Ok(crate::Checksum::from_raw(digest) == manifest_checksum),
569 Err(e) if is_environmental(&e) => Err(e),
570 Err(_) => Ok(false),
571 }
572 };
573 // The temp is condemned as an abandoned build ONLY when the original
574 // proves itself the file the manifest names. Otherwise neither copy is
575 // proven and the ambiguity surfaces — deleting the temp here could
576 // destroy a committed replacement whose swap never ran, and the original
577 // beside it is the damaged pre-repair source.
578 let condemn_only_if_proven = |ambiguity: crate::Error| -> crate::Result<bool> {
579 if original_is_authoritative()? {
580 Ok(false)
581 } else {
582 Err(ambiguity)
583 }
584 };
585 // A temp that READS but does not match is as ambiguous as one that does
586 // not read: it is either an abandoned build or a committed replacement
587 // that rotted after its manifest went durable.
588 let digest_mismatch = |digest: u128| crate::Error::ChecksumMismatch {
589 got: crate::Checksum::from_raw(digest),
590 expected: manifest_checksum,
591 };
592 let Some(bound) = restriction else {
593 return match compute_table_checksum(&**fs, tmp_path) {
594 Ok(digest) if crate::Checksum::from_raw(digest) == manifest_checksum => Ok(true),
595 Ok(digest) => condemn_only_if_proven(digest_mismatch(digest)),
596 Err(e) if is_environmental(&e) => Err(e),
597 Err(temp_err) => condemn_only_if_proven(temp_err),
598 };
599 };
600 let table = match crate::table::Table::recover(repair_recover_params(
601 config,
602 tmp_path.to_path_buf(),
603 manifest_checksum,
604 table_id,
605 Arc::clone(fs),
606 // This open exists only to locate the temp's punch offset from its own
607 // index; no entry is read through it, so the ingest offset is moot.
608 None,
609 )) {
610 Ok(table) => table,
611 Err(e) if is_environmental(&e) => return Err(e),
612 // A temp that does not open is USUALLY the mid-build crash — but the
613 // manifest may equally describe a COMMITTED replacement this run
614 // failed to swap, and then discarding it destroys the only published
615 // copy. Same proof as the checksum paths: condemn the temp only when
616 // the original proves itself the file the manifest names.
617 Err(e) => return condemn_only_if_proven(e),
618 };
619 let punch_offset = match table.punch_offset_for(bound.as_ref()) {
620 Ok(offset) => offset,
621 Err(e) if is_environmental(&e) => return Err(e),
622 Err(e) => return condemn_only_if_proven(e),
623 };
624 match compute_table_checksum_from(&**fs, tmp_path, punch_offset) {
625 Ok(digest) if crate::Checksum::from_raw(digest) == manifest_checksum => Ok(true),
626 Ok(digest) => condemn_only_if_proven(digest_mismatch(digest)),
627 Err(e) if is_environmental(&e) => Err(e),
628 Err(temp_err) => condemn_only_if_proven(temp_err),
629 }
630}
631
632/// The trustworthy EXACT restriction bound for `table_id`, or `None` when the
633/// scan has none and must fall back to the punch geometry.
634///
635/// One question, one answer, for both salvage arms: the arm that still holds a
636/// recovered `Table` and the arm whose whole-file recovery failed ask it
637/// identically, and a bound from either source is honored the same way — the
638/// only difference is that one reopens the table on it while the other
639/// re-imposes it on the salvaged replacement.
640///
641/// The clean manifest decides in both directions (see [`ManifestRestriction`]);
642/// only an `Unknown` manifest consults the sidecar mirror, and there a
643/// TRANSIENT read propagates while a missing / id-mismatched / corrupt one
644/// leaves no trustworthy bound.
645///
646/// # Errors
647///
648/// The digest a RESTRICTED view of `table_path` records: its live suffix from
649/// the punch offset `bound` falls at, which is what the manifest holds for a
650/// tight-space-punched table (see [`crate::table::Table::reopen_restricted`]).
651///
652/// `None` when the file cannot be opened or the bound cannot be located in its
653/// index: nothing to compare, never a claim of mismatch.
654///
655/// # Errors
656///
657/// Propagates an ENVIRONMENTAL failure; anything else answers `None`.
658#[cfg(feature = "std")]
659fn restricted_suffix_digest(
660 config: &Config,
661 fs: &Arc<dyn crate::fs::Fs>,
662 table_path: &std::path::Path,
663 table_id: TableId,
664 bound: &crate::UserKey,
665) -> crate::Result<Option<crate::Checksum>> {
666 // Opening reads the trailer, meta and index only; the data blocks the
667 // digest streams are read once, below.
668 let table = match Table::recover(repair_recover_params(
669 config,
670 table_path.to_path_buf(),
671 crate::Checksum::from_raw(0),
672 table_id,
673 Arc::clone(fs),
674 None,
675 )) {
676 Ok(table) => table,
677 Err(e) if is_environmental(&e) => return Err(e),
678 Err(_) => return Ok(None),
679 };
680 match table.suffix_checksum_for(Some(bound)) {
681 Ok(d) => Ok(Some(d)),
682 Err(e) if is_environmental(&e) => Err(e),
683 Err(_) => Ok(None),
684 }
685}
686
687/// Propagates an environmental sidecar read failure, so a retry re-reads it.
688#[cfg(feature = "std")]
689fn trustworthy_restriction_bound(
690 config: &Config,
691 fs: &dyn crate::fs::Fs,
692 table_path: &std::path::Path,
693 table_id: TableId,
694 manifest_restriction: &ManifestRestriction,
695) -> crate::Result<Option<crate::UserKey>> {
696 match manifest_restriction {
697 ManifestRestriction::Restricted(bound) => Ok(Some(bound.clone())),
698 ManifestRestriction::Unrestricted => Ok(None),
699 ManifestRestriction::Unknown => {
700 match crate::restrict_bound::read(fs, table_path, config.encryption.as_deref()) {
701 Ok(crate::restrict_bound::SidecarRead::Present(id, b)) if id == table_id => {
702 Ok(Some(b.into()))
703 }
704 Err(e) if is_environmental(&e) => Err(e),
705 Ok(_) | Err(_) => Ok(None),
706 }
707 }
708 }
709}
710
711/// The dictionary a blob file's own compression descriptor names, resolved
712/// against the tree's set; `None` for a file that uses no dictionary.
713///
714/// A blob file records ONE descriptor, so this answers with one dictionary; the
715/// set is what turns the recorded id back into bytes. Resolving here rather than
716/// passing the configured dictionary is what lets a file written under an
717/// earlier dictionary still be read and salvaged.
718///
719/// # Errors
720///
721/// [`crate::Error::ZstdDictMismatch`] naming the id when the tree does not hold
722/// it. The file is then intact bytes behind a missing context: decoding it
723/// without the dictionary would grade it as damage and drop every value in it,
724/// so the caller either stops the repair, which the operator answers by
725/// supplying the dictionary, or leaves a file nothing references to the sweep.
726#[cfg(all(feature = "std", zstd_any))]
727fn blob_file_dictionary(
728 config: &Config,
729 compression: crate::CompressionType,
730) -> crate::Result<Option<Arc<crate::compression::ZstdDictionary>>> {
731 config
732 .current_zstd_dictionaries()
733 .for_compression(compression)
734}
735
736/// What only the manifest knows of a table's place in the tree: its bulk-ingest
737/// sequence offset and its L0 recency. Both come from the ingestion's install,
738/// after its file was written, so neither can be read back from the file.
739#[cfg(feature = "std")]
740#[derive(Clone, Copy, Debug)]
741struct ManifestPosition {
742 global_seqno: SeqNo,
743 recency: TableId,
744}
745
746#[cfg(feature = "std")]
747impl ManifestPosition {
748 fn of_record(record: &crate::version::recovery::RecoveredTable) -> Self {
749 Self {
750 global_seqno: record.global_seqno,
751 recency: record.recency,
752 }
753 }
754
755 fn of_table(table: &Table) -> Self {
756 Self {
757 global_seqno: table.global_seqno(),
758 recency: table.l0_recency(),
759 }
760 }
761}
762
763/// Recover params for a repair's TRANSIENT table open: the tree's configured
764/// comparator / crypto / dictionary context (so the table decodes consistently
765/// with how it was written), and everything else neutral — tree id 0 and no
766/// descriptor table keep the open from polluting shared caches keyed by the
767/// real tree id.
768///
769/// `position` is EXPLICIT, never defaulted: a bulk-ingested SST keeps its
770/// entries at local seqno 0 and relies on the manifest-only offset for its
771/// effective MVCC ordering, so silently opening at 0 would mis-order and
772/// over-expose them. `None` means "no position is recoverable here", which is
773/// correct only for a manifest-loss rebuild; a zero offset is genuine (the
774/// first ingestion on a fresh counter commits it), not a stand-in for absence.
775#[cfg(feature = "std")]
776fn repair_recover_params(
777 config: &Config,
778 file_path: PathBuf,
779 checksum: crate::Checksum,
780 table_id: TableId,
781 fs: Arc<dyn crate::fs::Fs>,
782 position: Option<ManifestPosition>,
783) -> crate::table::RecoverParams {
784 let mut params = crate::table::RecoverParams::new(
785 file_path,
786 checksum,
787 table_id,
788 fs,
789 config.comparator.clone(),
790 config.cache.clone(),
791 );
792 if let Some(position) = position {
793 params.global_seqno = position.global_seqno;
794 params.recency = Some(position.recency);
795 }
796 params.encryption.clone_from(&config.encryption);
797 #[cfg(zstd_any)]
798 {
799 params.zstd_dictionaries = config.current_zstd_dictionaries();
800 }
801 params
802}
803
804/// [`repair_recover_params`] for a table the repair holds until it publishes
805/// the rebuilt manifest: its file goes through the configured descriptor
806/// cache, within its budget, so the tables held at once need no descriptor
807/// each. A tree id of its own: another copy of this table id, or its salvage
808/// replacement, must never be handed this file's descriptor.
809#[cfg(feature = "std")]
810fn held_recover_params(
811 config: &Config,
812 file_path: PathBuf,
813 checksum: crate::Checksum,
814 table_id: TableId,
815 fs: Arc<dyn crate::fs::Fs>,
816 position: Option<ManifestPosition>,
817) -> crate::table::RecoverParams {
818 let mut params = repair_recover_params(config, file_path, checksum, table_id, fs, position);
819 if let Some((cache, tree_id)) = held_descriptors(config) {
820 params.descriptor_table = Some(cache);
821 params.tree_id = tree_id;
822 }
823 params
824}
825
826/// The descriptors a repair cached, from its first tree id on, swept from the
827/// cache when the repair ends.
828#[cfg(feature = "std")]
829struct HeldDescriptors {
830 cache: Option<Arc<crate::DescriptorTable>>,
831 first: crate::tree::inner::TreeId,
832}
833
834#[cfg(feature = "std")]
835impl Drop for HeldDescriptors {
836 fn drop(&mut self) {
837 if let Some(cache) = &self.cache {
838 cache.remove_trees_from(self.first);
839 }
840 }
841}
842
843/// Lets go of a table the repair held before its file is removed, its cached
844/// descriptor with it: a backend that refuses to unlink an open file
845/// (Windows) would otherwise fail the removal while the cache keeps it open.
846#[cfg(feature = "std")]
847fn release_held(config: &Config, table: Table) {
848 if let Some(cache) = &config.descriptor_table {
849 cache.remove_for_table(&table.global_id());
850 }
851 drop(table);
852}
853
854/// The descriptor cache a table the repair holds keeps its file in, with a
855/// tree id of its own for it; `None` when the configuration keeps none.
856#[cfg(feature = "std")]
857fn held_descriptors(
858 config: &Config,
859) -> Option<(Arc<crate::DescriptorTable>, crate::tree::inner::TreeId)> {
860 config
861 .descriptor_table
862 .clone()
863 .map(|cache| (cache, crate::tree::inner::get_next_tree_id()))
864}
865
866/// Whether an I/O failure must PROPAGATE out of the repair instead of grading
867/// the file it came from — either UNAMBIGUOUSLY TRANSIENT, or an
868/// ENVIRONMENTAL access failure that does not implicate the bytes on disk.
869///
870/// The allowlist is deliberately narrow: `Interrupted` (`EINTR`) and
871/// `WouldBlock` (`EAGAIN`) — the interrupted-syscall errors a retry genuinely
872/// clears, and which a corrupt on-disk structure can NEVER produce — plus
873/// `PermissionDenied` (`EACCES` / `EPERM`): an ACL / ownership mistake the
874/// operator fixes, while grading the healthy file unreadable commits a
875/// manifest that excludes it and then REMOVES it, turning a recoverable
876/// configuration error into permanent data loss.
877///
878/// `Other` is NOT on the list, even though an injected fault or a raw `EIO`
879/// lands there, because a STRUCTURAL corruption lands there too: a corrupt
880/// trailer that decodes to a bad offset makes the reader seek before the start of
881/// the file, which Windows reports as `ERROR_NEGATIVE_SEEK` — an unmapped OS
882/// error the `From<std::io::Error>` bridge folds into `ErrorKind::Other`. Treating
883/// `Other` as propagating would then abort the WHOLE repair (blocking recovery of
884/// every healthy sibling table) on a single genuinely-corrupt SST, and the class
885/// is platform-dependent (the same corruption reads back `InvalidInput` on Unix).
886/// A hardware `EIO` is likewise usually a persistent bad-sector failure, so
887/// recording that table unreadable — while the rest recover — is the right
888/// outcome. Fault-injection tests therefore inject `Interrupted` to model a
889/// retryable fault.
890///
891/// This inspects the CRATE's [`crate::io::ErrorKind`], which is what a
892/// `crate::Error::Io` always carries.
893///
894/// The MIS-SUPPLIED RECOVERY CONTEXT errors are in the class too, for the
895/// same reason: they say the caller brought the wrong key or the wrong
896/// dictionary, not that the bytes rotted.
897///
898/// - [`crate::Error::Decrypt`]: an AEAD failure is exactly what a missing or
899/// wrong key produces on perfectly healthy ciphertext, and the two are
900/// cryptographically indistinguishable from genuine rot.
901/// - [`crate::Error::ZstdDictMismatch`]: the persisted descriptor names a
902/// dictionary the caller did not supply (`got: None`) or supplied a
903/// different one. The blob is intact; only the context is wrong.
904///
905/// Recording either as unreadable commits a manifest omitting the file —
906/// whose cleanup then DELETES it — turning a fixable configuration mistake
907/// into permanent loss. Propagating lets a re-run with the right context
908/// recover everything; on genuinely damaged bytes the repair fails instead of
909/// guessing, and they stay in place.
910fn is_environmental(e: &crate::Error) -> bool {
911 // The class itself lives on `Error` so the salvage paths classify
912 // identically — the two drifted apart once already, and the difference is
913 // invisible until a file is deleted over it.
914 e.is_environmental()
915}
916
917/// Whether manifest repair must fail closed on a table because its bulk-ingest
918/// sequence offset cannot be reconstructed from the SST alone (the rebuilt
919/// manifest would install it with offset 0 and silently mis-order / mis-expose
920/// its entries).
921///
922/// - `Some(true)`: authoritatively bulk-ingested — always fail closed.
923/// - `Some(false)`: a newer non-ingested table — safe (offset genuinely 0), even
924/// when its entries all sit at seqno 0 (a fresh tree's first batch).
925/// - `None`: a LEGACY SST written before the provenance flag existed — UNKNOWN.
926/// Treat it as bulk-ingested ONLY when its entries carry the ingest signature
927/// (present, and every LOCAL seqno 0), which a legacy bulk-ingest produces. A
928/// legacy first-batch-at-seqno-0 flush matches too and is conservatively
929/// dropped; the ambiguity is unavoidable without the flag.
930fn has_unrecoverable_ingest_offset(
931 bulk_ingested: Option<bool>,
932 item_count: u64,
933 max_local_seqno: crate::SeqNo,
934) -> bool {
935 match bulk_ingested {
936 Some(flagged) => flagged,
937 None => item_count > 0 && max_local_seqno == 0,
938 }
939}
940
941/// How faithfully a recovered candidate carries its source's content. Drives
942/// two DISTINCT report facts: anything but [`Complete`](Self::Complete)
943/// contributes a `lost_coverage` entry (scoped by the source's coverage), while
944/// only [`Salvaged`](Self::Salvaged) counts toward `RepairReport::salvaged` —
945/// a geometry-restricted original is lossy but was never block-salvaged, and
946/// it arises in plain (salvage-off) repairs too.
947#[derive(Copy, Clone, PartialEq, Eq)]
948enum Fidelity {
949 /// Every live row of the source is present.
950 Complete,
951 /// The ORIGINAL file reopened under a punch-GEOMETRY-derived restriction
952 /// (no exact sidecar, resurrection off): the bound is the straddling
953 /// block's END key, so that block's still-live suffix rows may be excluded.
954 GeometryRestricted,
955 /// A block-salvage rewrite: corrupt blocks (or blob records) were dropped.
956 Salvaged,
957}
958
959impl Fidelity {
960 fn is_complete(self) -> bool {
961 matches!(self, Self::Complete)
962 }
963}
964
965/// A recovered table plus its [`Fidelity`]. Repair keeps the best copy per
966/// table id, so a duplicate id in another table folder can supersede an
967/// earlier DAMAGED copy.
968///
969/// The physical location (`fs` / `path`) travels with the candidate so that when
970/// a duplicate SUPERSEDES it, the loser's file can be removed from `tables/`. The
971/// rebuilt manifest records only `id + checksum`, so two same-id files left in
972/// different folders would let recovery resolve the stale one by folder order and
973/// reopen it against the kept copy's mismatched checksum.
974struct TableCandidate {
975 table: Table,
976 fidelity: Fidelity,
977 fs: Arc<dyn crate::fs::Fs>,
978 path: PathBuf,
979 /// Whether this file's whole-file digest equals the one the committed
980 /// manifest records for its id. `false` also covers "cannot be compared":
981 /// no clean manifest, or a RESTRICTED table, whose entry holds the
982 /// live-suffix digest that this whole-file hash never reproduces.
983 matches_manifest: bool,
984}
985
986/// Records `candidate` for `id`, keeping the BETTER of the existing and the new
987/// copy: a COMPLETE recovery replaces a lossy salvage, so an intact duplicate in
988/// a later-scanned folder supersedes an earlier lossy one.
989///
990/// Two COMPLETE copies are not interchangeable either. Both open, both are
991/// self-consistent, and they can still hold DIFFERENT generations — a prior
992/// repair that retained a noncanonical copy while its cleanup left the older
993/// canonical file behind produces exactly that. Adopting the first-seen would
994/// re-stamp a fresh checksum over the stale bytes, and every surviving handle
995/// would then resolve against the wrong generation. The copy whose digest
996/// matches the committed manifest wins; with none to compare against (no clean
997/// manifest, or a restricted table whose entry digests only its live suffix)
998/// they really are equivalent and the first-seen stays.
999///
1000/// Two LOSSY copies are NOT equivalent. Damaged copies of one table in two
1001/// routed folders salvage independently, and one can recover far more of it
1002/// than the other; keeping the first-seen would discard rows the other salvage
1003/// did recover, which is avoidable loss. The fuller one wins, measured by the
1004/// entries its metadata records — the salvage writes exactly what it recovered,
1005/// so that count IS its completeness. Equal counts keep the first-seen, so the
1006/// choice stays deterministic across runs over the same directory.
1007///
1008/// Returns the DISPLACED loser (the rejected new candidate, or the superseded old
1009/// one) so the caller can record its file for removal; `None` when `id` was
1010/// previously unseen (nothing displaced).
1011#[must_use = "the displaced duplicate's file must be removed from tables/"]
1012fn keep_best_candidate(
1013 map: &mut crate::HashMap<TableId, TableCandidate>,
1014 id: TableId,
1015 candidate: TableCandidate,
1016) -> Option<TableCandidate> {
1017 let keeps_existing = match map.get(&id) {
1018 // Two completes: only the manifest can say which generation this id
1019 // is, so a copy that reproduces its digest displaces one that does
1020 // not. Neither matching (or nothing to match against) keeps the
1021 // incumbent, which is the previous first-seen rule.
1022 Some(existing) if existing.fidelity.is_complete() && candidate.fidelity.is_complete() => {
1023 existing.matches_manifest || !candidate.matches_manifest
1024 }
1025 // An existing COMPLETE copy is never displaced by a lossy one.
1026 Some(existing) if existing.fidelity.is_complete() => true,
1027 // A COMPLETE newcomer displaces a lossy incumbent.
1028 Some(_) if candidate.fidelity.is_complete() => false,
1029 // Both lossy: the one that recovered more of the table wins.
1030 Some(existing) => candidate.table.metadata.item_count <= existing.table.metadata.item_count,
1031 None => false,
1032 };
1033 if keeps_existing {
1034 return Some(candidate);
1035 }
1036 // The new candidate supersedes: `insert` returns the displaced old copy.
1037 map.insert(id, candidate)
1038}
1039
1040/// Removes a blob-salvage temp on a path where the repair goes on to COMMIT a
1041/// manifest. A missing temp is fine; ANY other failure fails the repair: the
1042/// rebuilt manifest never references the temp, so left in `blobs/` the next
1043/// open's orphan sweep would hit the same removal failure — reporting success
1044/// for a tree that cannot open would be a lie. Quarantine is NOT an out here:
1045/// it preserves damaged DATA for the operator, and a temp is discardable
1046/// garbage — while a directory that refuses removal refuses the rename too.
1047/// Paths where the repair itself returns an error need only best-effort
1048/// removal: the retry re-attempts it and fails honestly if it still cannot.
1049#[cfg(feature = "std")]
1050fn remove_temp(config: &Config, temp: &std::path::Path) -> crate::Result<()> {
1051 match config.fs.remove_file(temp) {
1052 Ok(()) => Ok(()),
1053 Err(e) if e.kind() == crate::io::ErrorKind::NotFound => Ok(()),
1054 Err(e) => {
1055 log::error!(
1056 "repair: cannot remove salvage temp {} ({e}); failing the repair — \
1057 left in place, the next open's orphan sweep would hit the same error",
1058 temp.display(),
1059 );
1060 Err(e.into())
1061 }
1062 }
1063}
1064
1065/// Whether `a` and `b` name the SAME physical file — a symlink, junction, or
1066/// case-insensitive alias resolving to one directory entry (two configured table
1067/// folders pointing at the same location). Used so a repeated sighting of one SST
1068/// through an alias is never removed as a "duplicate" (which would destroy the
1069/// kept copy and orphan the manifest entry).
1070///
1071/// The two candidates must live in the SAME filesystem namespace for a path
1072/// comparison to mean anything: a virtual (`MemFs`) table can sit at a path that
1073/// also exists on the host filesystem, and canonicalizing both spellings through
1074/// the host would call those distinct files aliases — the loser would then survive
1075/// and a later reopen could resolve that leftover against the kept copy's
1076/// manifest checksum. Backends advertise namespace identity through
1077/// [`Fs::backend_id`](crate::fs::Fs::backend_id), whose `None` means "no shared
1078/// namespace guarantee" and is therefore treated as DISTINCT.
1079///
1080/// Within one namespace, alias resolution is the backend's own
1081/// ([`Fs::same_file`](crate::fs::Fs::same_file)): kernel-backed backends
1082/// canonicalize through the host, virtual ones compare paths literally — a
1083/// host symlink must never alias two distinct virtual files. A probe that
1084/// cannot decide answers `false` (distinct), so a genuine duplicate is still
1085/// removed.
1086#[cfg(feature = "std")]
1087fn same_physical_file(
1088 fs_a: &dyn crate::fs::Fs,
1089 a: &std::path::Path,
1090 fs_b: &dyn crate::fs::Fs,
1091 b: &std::path::Path,
1092) -> crate::Result<bool> {
1093 match (fs_a.backend_id(), fs_b.backend_id()) {
1094 (Some(id_a), Some(id_b)) if id_a == id_b => {}
1095 // Two backends that each CLAIM an identity and disagree are provably
1096 // distinct namespaces: path spellings are not comparable.
1097 (Some(_), Some(_)) => return Ok(false),
1098 // A backend WITHOUT an identity claim (a custom `Fs` keeping the
1099 // default `None`): the SAME instance is trivially one namespace, so
1100 // its own probe below decides. Two DIFFERENT instances cannot be
1101 // proven distinct — and `false` here AUTHORIZES deleting what may be
1102 // an alias of the kept file — so the verdict is inconclusive and the
1103 // repair aborts for the operator to sort out.
1104 _ => {
1105 if !core::ptr::addr_eq(
1106 core::ptr::from_ref::<dyn crate::fs::Fs>(fs_a),
1107 core::ptr::from_ref::<dyn crate::fs::Fs>(fs_b),
1108 ) {
1109 return Err(crate::Error::Io(crate::io::Error::new(
1110 crate::io::ErrorKind::InvalidInput,
1111 "alias identity is inconclusive: the backend reports no namespace id",
1112 )));
1113 }
1114 }
1115 }
1116 // Alias resolution belongs to the BACKEND, not the host: canonicalizing a
1117 // virtual backend's paths through the host filesystem would let a host
1118 // symlink alias two unrelated virtual files, and the "duplicate" loser
1119 // would survive. Kernel-backed backends canonicalize; virtual ones compare
1120 // literally. An INCONCLUSIVE probe propagates: `false` authorizes
1121 // duplicate deletion, and deleting what may be an alias of the kept copy
1122 // unlinks the very directory entry the rebuilt manifest retains.
1123 Ok(fs_a.same_file(a, b)?)
1124}
1125
1126/// Records a duplicate table file that lost to a better same-id copy: reported
1127/// as considered-but-not-referenced, and removed once the manifest is durable so
1128/// recovery can never resolve it instead of the kept copy.
1129#[cfg(feature = "std")]
1130fn discard_duplicate(
1131 loser: TableCandidate,
1132 unreadable_files: &mut Vec<(PathBuf, String)>,
1133 redundant_unreadable: &mut crate::HashSet<PathBuf>,
1134 discard_after_commit: &mut Vec<(Arc<dyn crate::fs::Fs>, PathBuf, String)>,
1135) {
1136 let TableCandidate {
1137 table, fs, path, ..
1138 } = loser;
1139 drop(table); // release the open file handle
1140 let reason = "duplicate table id superseded by another copy";
1141 discard_after_commit.push((fs, path.clone(), reason.to_string()));
1142 // Reported, but NOT a loss: the copy that displaced it holds the same id,
1143 // so nothing needs replaying. Counting it would answer `FullHistory` and
1144 // send an external WAL over a tree that lost nothing.
1145 redundant_unreadable.insert(path.clone());
1146 unreadable_files.push((path, reason.to_string()));
1147}
1148
1149/// Builds a [`TableCandidate`] from a recovered table plus its physical location,
1150/// records it as the best copy for `id`, and marks any duplicate displaced by
1151/// the decision for post-commit removal. The one path every recovered table
1152/// takes to enter the manifest, so a superseded same-id file is never left
1153/// discoverable once the rebuild is durable.
1154/// Records a finished SALVAGE replacement and queues the post-commit swap
1155/// that publishes it.
1156///
1157/// Both salvage arms — the one whose VERIFICATION failed and the one whose
1158/// whole-file RECOVERY failed — reach exactly this state, and they must not
1159/// drift apart: the replacement enters the manifest through `record_best` like
1160/// any recovered table, and the swap carries whether it is RESTRICTED so a
1161/// retried swap can tell an already-moved replacement sidecar from a stale
1162/// source one (see `commit_repair_tmp`).
1163#[cfg(feature = "std")]
1164#[expect(
1165 clippy::too_many_arguments,
1166 reason = "location + report threaded through"
1167)]
1168fn keep_salvaged_replacement(
1169 map: &mut crate::HashMap<TableId, TableCandidate>,
1170 unreadable_files: &mut Vec<(PathBuf, String)>,
1171 redundant_unreadable: &mut crate::HashSet<PathBuf>,
1172 discard_after_commit: &mut Vec<(Arc<dyn crate::fs::Fs>, PathBuf, String)>,
1173 swap_after_commit: &mut Vec<(Arc<dyn crate::fs::Fs>, PathBuf, PathBuf, bool)>,
1174 id: TableId,
1175 table: Table,
1176 fs: &Arc<dyn crate::fs::Fs>,
1177 table_path: &std::path::Path,
1178 output_path: PathBuf,
1179) -> crate::Result<()> {
1180 let restricted = table.restrict_lower_bound().is_some();
1181 // Queue the swap FIRST, then let `record_best` decide: if this candidate
1182 // loses (rejected outright, or displaced later by an intact duplicate from
1183 // another routed folder), that same call drops the swap again, so only a
1184 // replacement the rebuilt manifest actually references is ever renamed.
1185 swap_after_commit.push((
1186 Arc::clone(fs),
1187 output_path,
1188 table_path.to_path_buf(),
1189 restricted,
1190 ));
1191 record_best(
1192 map,
1193 unreadable_files,
1194 redundant_unreadable,
1195 discard_after_commit,
1196 swap_after_commit,
1197 id,
1198 table,
1199 Fidelity::Salvaged,
1200 fs,
1201 table_path,
1202 // A salvage's copy is a fresh file the manifest never digested.
1203 false,
1204 )
1205}
1206
1207#[cfg(feature = "std")]
1208#[expect(
1209 clippy::too_many_arguments,
1210 reason = "location + report threaded through"
1211)]
1212fn record_best(
1213 map: &mut crate::HashMap<TableId, TableCandidate>,
1214 unreadable_files: &mut Vec<(PathBuf, String)>,
1215 redundant_unreadable: &mut crate::HashSet<PathBuf>,
1216 discard_after_commit: &mut Vec<(Arc<dyn crate::fs::Fs>, PathBuf, String)>,
1217 // Queued swaps, so a candidate that LOSES here takes its own swap with it:
1218 // a replacement the rebuilt manifest does not reference must not be renamed
1219 // onto a name the manifest gives to another file, and a rename fault on a
1220 // disposable temp would otherwise turn a good repair into
1221 // `RepairedButUnopened`. A swap is keyed by the destination it publishes,
1222 // which is exactly the losing candidate's own path.
1223 swap_after_commit: &mut Vec<(Arc<dyn crate::fs::Fs>, PathBuf, PathBuf, bool)>,
1224 id: TableId,
1225 table: Table,
1226 fidelity: Fidelity,
1227 fs: &Arc<dyn crate::fs::Fs>,
1228 path: &std::path::Path,
1229 matches_manifest: bool,
1230) -> crate::Result<()> {
1231 let candidate = TableCandidate {
1232 table,
1233 fidelity,
1234 fs: Arc::clone(fs),
1235 path: path.to_path_buf(),
1236 matches_manifest,
1237 };
1238 if let Some(loser) = keep_best_candidate(map, id, candidate) {
1239 // If the displaced copy physically ALIASES the kept one (same directory
1240 // entry via a symlink / junction / case-insensitive path), it is the SAME
1241 // file — removing it would destroy the kept copy and orphan the manifest
1242 // entry. Drop the loser's handle in place instead. An INCONCLUSIVE
1243 // probe aborts the repair for a retry: guessing "distinct" would
1244 // authorize exactly that destruction.
1245 let is_alias = match map.get(&id) {
1246 Some(kept) => same_physical_file(&*loser.fs, &loser.path, &*kept.fs, &kept.path)?,
1247 None => false,
1248 };
1249 // The loser's replacement (if it built one) is never published — and
1250 // dropping the swap drops the ONLY reference to that `{id}.repair-tmp`,
1251 // so it is re-queued for removal instead of silently left behind.
1252 // Left in place it is exactly the shape the next open cannot resolve:
1253 // a temp whose digest does not match the manifest, with no original
1254 // beside it in THAT folder to prove it abandoned — so the open fails
1255 // before it ever reaches the healthy copy this repair kept.
1256 let mut orphaned_temps = Vec::new();
1257 swap_after_commit.retain(|(fs, temp, dest, _)| {
1258 let publishes_loser = dest == &loser.path;
1259 if publishes_loser {
1260 orphaned_temps.push((Arc::clone(fs), temp.clone()));
1261 }
1262 !publishes_loser
1263 });
1264 for (fs, temp) in orphaned_temps {
1265 discard_after_commit.push((
1266 fs,
1267 temp,
1268 "unpublished replacement of a displaced duplicate".to_string(),
1269 ));
1270 }
1271 if is_alias {
1272 return Ok(());
1273 }
1274 discard_duplicate(
1275 loser,
1276 unreadable_files,
1277 redundant_unreadable,
1278 discard_after_commit,
1279 );
1280 }
1281 Ok(())
1282}
1283
1284/// Block-salvages a corrupt SST during repair: reads the UNTOUCHED original in
1285/// place, writes a fresh SST holding its recoverable blocks to `table_path` (the
1286/// unpublished `{id}.repair-tmp`), and reopens it.
1287///
1288/// Returns `Ok(None)` when nothing was recoverable (the source is untouched and
1289/// no replacement exists), or `Err` when even salvage cannot open the source
1290/// (its metadata / index / SFA trailer is itself unreadable).
1291/// Whether a freshly-recovered SST passes the salvage-mode block verify.
1292///
1293/// One uniform path for encrypted and unencrypted tables: the out-of-band
1294/// section walk. Block headers and payload checksums are PLAINTEXT, so the
1295/// walk needs the provider only to decode the meta block (the per-SST ECC
1296/// descriptor); every section — data, index/TLI, filter, zone map, delete
1297/// bitmap, locator, meta — is then verified against its raw on-disk checksum,
1298/// which flags even a persistent ECC-CORRECTABLE fault (a live read would
1299/// silently heal it in memory while the corrupt bytes stay on disk).
1300/// Classifies a block-verifier result for the salvage gate. A structural
1301/// divergence (a checksum / decode / cross-check mismatch) is genuine
1302/// corruption: `Ok(true)`, route the table through salvage. Only a TRANSIENT
1303/// [`crate::Error::Io`] (the [`is_environmental`] allowlist) aborts the repair
1304/// (`Err`) for a retry, rather than dropping a healthy block into a partial
1305/// replacement. A PERSISTENT I/O failure is NOT retryable — a bad sector, or a
1306/// structural corruption surfacing as `Io(Other)` on some platforms — so it is
1307/// graded as corruption and salvaged too, rather than aborting the whole repair
1308/// and stranding every other healthy table on one unrecoverable read.
1309fn is_corruption(res: crate::Result<()>) -> crate::Result<bool> {
1310 match res {
1311 Ok(()) => Ok(false),
1312 Err(e) if is_environmental(&e) => Err(e),
1313 Err(_) => Ok(true),
1314 }
1315}
1316
1317fn block_verify_verdict(
1318 config: &Config,
1319 folder_fs: &Arc<dyn crate::fs::Fs>,
1320 table_path: &std::path::Path,
1321 table: &Table,
1322) -> crate::Result<BlockVerifyVerdict> {
1323 // Walk only the recovered view's LIVE data: for a tight-space RESTRICTED view
1324 // (a valid `.restrict-bound` sidecar was accepted) the `[0, punch_offset)`
1325 // prefix is hole-punched and reads as zeros, so starting at byte 0 would
1326 // report those dead blocks as corruption and repair would then drop an
1327 // otherwise-healthy restricted SST. `punch_offset()` is `0` for a normal table.
1328 let data_start = table.punch_offset()?;
1329 let report = crate::verify::verify_sst_file_with_context(
1330 folder_fs,
1331 table_path,
1332 config.encryption.as_ref(),
1333 // Repair KNOWS the durable id (recovery already cross-checked it
1334 // against the file name), so the verify probe enforces the same meta
1335 // id check — a checksum-clean forged tail meta falls back to the
1336 // intact MID mirror instead of dictating a forged ECC descriptor.
1337 Some(table.metadata.id),
1338 data_start,
1339 );
1340 // A TRANSIENT read failure DURING the walk (a retryable `Interrupted` /
1341 // `WouldBlock`) is not block corruption — routing it through salvage would
1342 // re-read the same bytes and drop a healthy block — and neither is an
1343 // ENVIRONMENTAL `PermissionDenied` (an ACL / ownership mistake the operator
1344 // fixes; the bytes on disk are not implicated). Propagate both so the repair
1345 // aborts and the operator retries, mirroring the decode-load gate below. Any
1346 // OTHER kind falls through to the corruption verdict: a truncation
1347 // (`UnexpectedEof`) is genuine on-disk damage, and a PERSISTENT failure
1348 // (`Other` / EIO) is not fixed by a retry, so aborting forever would strand
1349 // every healthy sibling table on one bad SST.
1350 // This matches the `is_corruption` allowlist policy exactly.
1351 //
1352 // This gate depends on the walk CLASSIFYING transient faults as one of these
1353 // two I/O-bearing variants: a mid-walk seek failure, a transient block-header
1354 // read, and a raw-section read all surface as `DataReadError` rather than
1355 // being folded into `HeaderCorrupted` / `TocCorrupted`, so a flaky read here
1356 // is not mistaken for corruption and salvaged.
1357 for e in &report.errors {
1358 if let crate::verify::BlockVerifyError::SstFileUnreadable { error, .. }
1359 | crate::verify::BlockVerifyError::DataReadError { error, .. } = e
1360 && error.kind().is_environmental()
1361 {
1362 // Preserve the ErrorKind: re-wrapping as `Other` would make the
1363 // caller's `is_environmental` check see a non-propagating kind
1364 // and re-grade this retryable / environmental failure as
1365 // corruption, defeating the propagation intent of this very gate.
1366 return Err(crate::Error::Io(crate::io::Error::new(
1367 error.kind(),
1368 error.to_string(),
1369 )));
1370 }
1371 }
1372
1373 // A non-parity error is corruption regardless of any warnings.
1374 let verdict = if !report
1375 .errors
1376 .iter()
1377 .all(|e| matches!(e, crate::verify::BlockVerifyError::EccParityMismatch { .. }))
1378 {
1379 BlockVerifyVerdict::Corrupt
1380 } else if report
1381 .warnings
1382 .iter()
1383 .any(|w| matches!(w, crate::verify::BlockVerifyWarning::UnrecognizedEcc { .. }))
1384 {
1385 // Unrecognized ECC descriptor: the walk SKIPPED the SST-block
1386 // sections entirely (their trailer length is underivable), so
1387 // NOTHING about the data was verified — a stronger degradation
1388 // than a checked-but-unverifiable-parity report. Graded BEFORE the
1389 // parity-only arm below: parity mismatches in the still-walked
1390 // self-describing meta blocks must not mask the skipped data /
1391 // index sections.
1392 BlockVerifyVerdict::DegradedUnscanned
1393 } else if is_corruption(table.verify_blob_links())? {
1394 // Same reasoning for the blob-link list: the section carries no
1395 // per-section checksum, so the walk can only validate its SHAPE — a
1396 // flipped blob id passes it. Cross-check against the table's own
1397 // indirection entries (a no-op without the section); a mismatch is
1398 // corruption, and salvage derives the links from the recovered
1399 // indirections rather than copying the forged list.
1400 BlockVerifyVerdict::Corrupt
1401 } else if is_corruption(table.verify_tli_mirrors())? {
1402 // Each TLI mirror is independently checksum-clean to the walk, but a
1403 // forged copy that DECODES to a different handle list would steer
1404 // the next recovery (which prefers the tail) away from real blocks.
1405 // Diverging decoded mirrors are corruption; salvage walks the HEAD
1406 // copy, so the recovered SST is rebuilt from a single, fully
1407 // re-verified handle list. BOTH mirrors forged to the SAME list that
1408 // OMITS a physical block are covered too: the salvage walk
1409 // cross-checks the index against the physical data-section tiling
1410 // and frames the uncovered bytes from their block headers, so the
1411 // hidden block is recovered (or reported dropped), never silently
1412 // missing from an apparently complete copy.
1413 BlockVerifyVerdict::Corrupt
1414 } else if is_corruption(table.verify_block_layout())? {
1415 // A checksum-clean block_layout re-stamped to another structurally
1416 // valid boundary set mis-maps the partial range-read path's
1417 // decompression bounds, silently omitting keys. Boundaries that
1418 // disagree with the frames' actual inner blocks are corruption;
1419 // salvage re-derives the layout when re-encoding.
1420 BlockVerifyVerdict::Corrupt
1421 } else if is_corruption(
1422 table
1423 .verify_reconcile_gates(config.prefix_extractor.as_ref(), false)
1424 .map_err(|(_, e)| e),
1425 )? {
1426 // The semantic cross-checks, all on ONE decode of each live block.
1427 // Every one of them catches a section that is checksum-clean to the
1428 // out-of-band walk yet lies about the entries, and every one routes
1429 // the table to salvage, which rebuilds the section from the re-emitted
1430 // data:
1431 //
1432 // - per-KV footers: a stale digest behind a re-stamped block checksum.
1433 // Graded BEFORE the degradation arms below — a forged footer also
1434 // leaves the parity trailer mismatched, and grading that as
1435 // "parity-only degradation" would retain a table with a KNOWN-stale
1436 // entry digest.
1437 // - seqno bounds: re-stamped to another structurally valid map, which
1438 // `scan_since_seqno` trusts to SKIP blocks.
1439 // - entry counts: a valid prefix followed by a malformed tail decodes
1440 // short while the trailer still declares the full count.
1441 // - zone map: forged min/max let a predicate scan skip blocks holding
1442 // matching rows.
1443 // - locator: re-stamped to resolve a key to a block other than its
1444 // newest-version one, so point_read returns a stale value without
1445 // falling back to the sorted index.
1446 // - filter: an existing key turned into a false negative disappears
1447 // from every read.
1448 // - point-read reachability: a hidden hash bucket or a misdirected
1449 // offset makes point_read miss data the block still decodes.
1450 // - metadata bounds: a narrowed key range hides real keys (and the
1451 // range tombstones masking older tables) from run selection.
1452 BlockVerifyVerdict::Corrupt
1453 } else if !report.is_ok() {
1454 // Parity-ONLY rot: every payload checksum verified clean, only the
1455 // recovery margin is dead. The data is fully readable, so it grades
1456 // like a warning-bearing report — salvage preferred (the rewrite
1457 // regenerates fresh parity), but never at the cost of dropping data
1458 // salvage cannot re-emit.
1459 BlockVerifyVerdict::DegradedButReadable
1460 } else if report.has_warnings() {
1461 // Everything scanned verified clean, but the parity trailers could
1462 // not be recomputed (a parity-less build). The caller decides
1463 // between salvage (a rewrite under fully-verifiable framing) and
1464 // keeping the table when salvage cannot re-emit it.
1465 BlockVerifyVerdict::DegradedButReadable
1466 } else {
1467 BlockVerifyVerdict::Clean
1468 };
1469 Ok(verdict)
1470}
1471
1472/// Outcome of the salvage-mode block verify, from the repair gate's point of
1473/// view (see [`block_verify_verdict`]).
1474enum BlockVerifyVerdict {
1475 /// Every section verified against its raw on-disk checksum.
1476 Clean,
1477 /// Every payload the walk checked verified clean, but the table is
1478 /// DEGRADED: its parity trailers rotted or could not be recomputed while
1479 /// the payloads stayed intact. Prefer a salvage rewrite, but never at
1480 /// the cost of dropping data salvage cannot re-emit.
1481 DegradedButReadable,
1482 /// The walk could not scan the SST-block sections at all (an
1483 /// unrecognized ECC descriptor): the data is UNVERIFIED, not merely
1484 /// degraded — any keep decision must first verify it another way.
1485 DegradedUnscanned,
1486 /// At least one payload / section failed verification.
1487 Corrupt,
1488}
1489
1490/// What the repair should do with a freshly-recovered table, based on the
1491/// salvage-mode block verify.
1492#[derive(Debug)]
1493enum RepairKeepDecision {
1494 /// The table joins the rebuilt manifest as-is.
1495 Keep,
1496 /// The table is routed through block salvage: its readable blocks are
1497 /// rewritten into a fresh copy beside it.
1498 Salvage,
1499 /// The table can be neither trusted nor faithfully salvaged under the
1500 /// active resurrection policy: it is EXCLUDED from the rebuilt manifest,
1501 /// with this reason, and its file removed once that manifest is durable.
1502 /// The tree still opens. Its rows come back through recompaction, a replica,
1503 /// a checkpoint plus journal replay, or a backup — and, where the reason
1504 /// says so, from a repair run WITH resurrection enabled instead.
1505 Drop(&'static str),
1506}
1507
1508/// Whether the on-disk TOC catalogue could HIDE a deletion section — see
1509/// [`crate::verify::toc_may_hide_deletion_section`]. A STRUCTURAL catalogue
1510/// ambiguity grades `Ok(true)` (fail closed): if the catalogue cannot be parsed
1511/// to prove no section is hidden, salvage must not trust the parsed absence of
1512/// deletion metadata.
1513///
1514/// # Errors
1515///
1516/// Propagates a TRANSIENT [`crate::Error::Io`] from opening or reading the
1517/// trailer. Grading a retryable read as `true` would send a table
1518/// [`repair_with_salvage`](Self) already found corrupt to `Quarantine` — dropping
1519/// its healthy ranges from the rebuilt manifest — when a retry of the probe could
1520/// have let block salvage recover them.
1521pub(crate) fn toc_may_hide_deletions(
1522 folder_fs: &Arc<dyn crate::fs::Fs>,
1523 table_path: &std::path::Path,
1524) -> crate::Result<bool> {
1525 let mut file = match folder_fs.open(table_path, &crate::fs::FsOpenOptions::new().read(true)) {
1526 Ok(file) => file,
1527 // A TRANSIENT open failure propagates (a retry could open the file and
1528 // prove no hidden section); a PERSISTENT one fails closed as catalogue
1529 // ambiguity — we cannot read the TOC to prove it hides no deletion
1530 // section, so `true` drops the table rather than resurrecting masked rows.
1531 Err(e) => {
1532 let err = crate::Error::Io(e);
1533 return if is_environmental(&err) {
1534 Err(err)
1535 } else {
1536 Ok(true)
1537 };
1538 }
1539 };
1540 match crate::sfa::Reader::from_reader(&mut file) {
1541 Ok(reader) => Ok(crate::verify::toc_may_hide_deletion_section(
1542 reader.toc(),
1543 reader.toc_pos(),
1544 )),
1545 // A transient trailer read propagates (retry could prove no hidden
1546 // section); a persistent I/O failure or a structural trailer failure is
1547 // genuine catalogue ambiguity that fails closed.
1548 Err(crate::sfa::Error::Io(e)) => {
1549 let err = crate::Error::Io(e);
1550 if is_environmental(&err) {
1551 Err(err)
1552 } else {
1553 Ok(true)
1554 }
1555 }
1556 Err(_) => Ok(true),
1557 }
1558}
1559
1560/// Grades a freshly-recovered table into a [`RepairKeepDecision`].
1561///
1562/// `Corrupt` always salvages. `DegradedButReadable` (payloads verified clean,
1563/// only the parity trailers rotted or could not be recomputed) salvages ONLY
1564/// when salvage can faithfully re-emit the table: a range-tombstone SST is
1565/// rejected by the block walk, so routing it through salvage would drop
1566/// healthy, verified data over dead parity — it is kept as-is (with an
1567/// operator-facing warning) instead. `DegradedUnscanned` (unrecognized ECC
1568/// descriptor: the walk verified NOTHING about the data) never keeps: a
1569/// rewritable table salvages, and a range-tombstone table — which cannot be
1570/// verified in full (every lazy side structure would need its own
1571/// handle-based check) and cannot be re-emitted — is excluded (recompact under
1572/// a supported scheme to re-admit it) instead of riding unverified into the
1573/// rebuilt manifest.
1574///
1575/// `allow_resurrection` governs the one ambiguous case: a corrupt catalogue
1576/// that could conceal a deletion section. Off (default) excludes the table (its
1577/// visibility is unrecoverable, so admitting it would resurrect masked rows);
1578/// on, it salvages, accepting that suppressed rows reappear.
1579fn verify_keep_decision(
1580 config: &Config,
1581 folder_fs: &Arc<dyn crate::fs::Fs>,
1582 table_path: &std::path::Path,
1583 table: &Table,
1584 allow_resurrection: bool,
1585 // Whether this repair may REWRITE a damaged table (block salvage). Off,
1586 // the degraded verdicts resolve without a rewrite: corrupt / unverifiable
1587 // content is set aside (with the reason pointing at the salvage-enabled
1588 // repair), while rotted-parity-but-readable content is KEPT — its payloads
1589 // verified clean, and blessing its digest is the entry into the normal
1590 // attributable heal (a patrol re-stamps the parity and reconciles).
1591 salvage: bool,
1592) -> crate::Result<RepairKeepDecision> {
1593 Ok(
1594 match block_verify_verdict(config, folder_fs, table_path, table)? {
1595 BlockVerifyVerdict::Clean => RepairKeepDecision::Keep,
1596 BlockVerifyVerdict::Corrupt => {
1597 // A `Corrupt` verdict from a catalogue that could HIDE a deletion
1598 // section (an omitted / renamed / shadowed `range_tombstones` or
1599 // `delete_bitmap`) makes the table's visibility unrecoverable: the
1600 // positional salvage walk reopens the same forged TOC, sees no
1601 // deletion section in the parsed state, and re-emits the suppressed
1602 // rows as LIVE. The salvage-side resurrection guard only inspects
1603 // the PARSED deletion state, which the concealment defeats, so the
1604 // decision has to happen here, governed by the resurrection flag:
1605 // off, exclude the table (admitting it would resurrect masked
1606 // rows); on, salvage and accept the resurrection. A relabel that
1607 // keeps the tiling intact but re-roles the block is caught inside
1608 // salvage itself (`salvage_with_context` fails closed on a corrupt
1609 // rebuildable section when no deletion is visible), which both this
1610 // path and the recovery-failure salvage path funnel through.
1611 //
1612 // Probed BEFORE the salvage-off branch below on purpose: with
1613 // salvage off the table is dropped either way, but this reason
1614 // is the accurate one — pointing that operator at a
1615 // salvage-enabled repair would mislead (it drops the
1616 // concealment case too, unless resurrection is enabled).
1617 if toc_may_hide_deletions(folder_fs, table_path)? && !allow_resurrection {
1618 RepairKeepDecision::Drop(
1619 "TOC corruption may hide deletion metadata (range tombstones \
1620 / delete bitmap); its visibility is unrecoverable, so the table \
1621 is excluded to avoid resurrecting masked rows. Enable \
1622 resurrection to salvage it, accepting that suppressed rows \
1623 reappear",
1624 )
1625 } else if salvage {
1626 RepairKeepDecision::Salvage
1627 } else {
1628 RepairKeepDecision::Drop(
1629 "verification found corrupt data blocks; run a salvage-enabled \
1630 repair to rewrite the readable blocks",
1631 )
1632 }
1633 }
1634 BlockVerifyVerdict::DegradedButReadable => {
1635 if salvage && table.range_tombstones().is_empty() {
1636 RepairKeepDecision::Salvage
1637 } else {
1638 log::warn!(
1639 "table {} at {}: every payload verified clean but its ECC is \
1640 partially uncheckable or rotted, and this repair cannot rewrite it \
1641 (salvage off, or range tombstones it cannot re-emit) — keeping the \
1642 table as-is; a patrol heal or recompaction re-stamps it under \
1643 fresh, verifiable parity",
1644 table.metadata.id,
1645 table_path.display(),
1646 );
1647 RepairKeepDecision::Keep
1648 }
1649 }
1650 BlockVerifyVerdict::DegradedUnscanned => {
1651 if salvage && table.range_tombstones().is_empty() {
1652 RepairKeepDecision::Salvage
1653 } else if salvage {
1654 RepairKeepDecision::Drop(
1655 "ECC descriptor unrecognized (the block walk cannot verify the \
1656 table) and salvage cannot re-emit its range tombstones; the table \
1657 is excluded (recompact it under a supported scheme to re-admit it)",
1658 )
1659 } else {
1660 RepairKeepDecision::Drop(
1661 "ECC descriptor unrecognized (the block walk cannot verify the \
1662 table); run a salvage-enabled repair to rewrite it under fresh, \
1663 verifiable parity",
1664 )
1665 }
1666 }
1667 },
1668 )
1669}
1670
1671/// Outcome of [`try_salvage_table`].
1672enum SalvageOutcome {
1673 /// A clean replacement was written and reopened, ready to install.
1674 Salvaged(Table),
1675 /// Nothing was recoverable, or the replacement was rejected (an
1676 /// unreconstructible bulk-ingest offset); the caller records the table
1677 /// unreadable and removes its file once the manifest is durable.
1678 Unusable,
1679 /// The `reject_punched_without_bound` guard fired: a salvage-dropped data
1680 /// extent of the source reads as zeros (the hole-punch signature), so the
1681 /// source lost data to a punch whose bound is unrecoverable. The
1682 /// unrestricted replacement was rejected and removed — installing it would
1683 /// resurrect the reclaimed region's superseded rows.
1684 PunchedBoundLost,
1685}
1686
1687/// What one [`try_salvage_table`] call operates on: the source paths and the
1688/// per-call policy. Bundled so the salvage entry keeps a small signature as
1689/// its policy surface grows.
1690#[cfg(feature = "std")]
1691struct TableSalvage<'a> {
1692 /// The damaged original to read, UNTOUCHED and still at its own path. It is
1693 /// never displaced first: the copy goes to a fresh name beside it, and the
1694 /// source is removed only once the manifest names the copy. A salvage that
1695 /// fails therefore leaves the directory exactly as it was found, and the
1696 /// retry re-derives the same result from the same bytes.
1697 source: &'a std::path::Path,
1698 /// Where the recovered copy is written (a fresh, unused path).
1699 table_path: &'a std::path::Path,
1700 /// The durable table id (its file name).
1701 table_id: TableId,
1702 /// Fail closed when the salvage walk reveals the source was PUNCHED (a
1703 /// dropped data extent reads as zeros) — set by the recovery-failure arm
1704 /// when it has no recoverable restriction bound and resurrection is off.
1705 /// The pre-salvage first-bytes probe catches a punched FIRST block cheaply,
1706 /// but a partial punch (the punch-on-drop reclaim continues past an
1707 /// individual `punch_hole` failure) can leave the first block intact while
1708 /// later prefix blocks are zeroed; only the walk sees those. The
1709 /// verification arm passes `false`: it derives the bound from its restricted
1710 /// view, so its salvage output is re-restricted, never ambiguous.
1711 reject_punched_without_bound: bool,
1712 /// Per-blob handle rewrite for a table referencing a blob file this repair
1713 /// reshaped (salvaged into a compacted copy, or recovered with a punched
1714 /// frontier); `None` on the plain corrupt-table salvage paths.
1715 blob_rewrite:
1716 Option<Arc<crate::HashMap<crate::vlog::BlobFileId, crate::salvage::BlobFileRewrite>>>,
1717 /// Cell rows of the table to make the owners of objects whose owning row
1718 /// the rebuild lost; `None` everywhere but the ownership pass.
1719 owner_promotions: Option<Arc<crate::salvage::OwnerPromotions>>,
1720 /// The source's RECOVERED manifest position, when it is known: from a
1721 /// clean manifest record, or from a source the scan already admitted (the
1722 /// blob-handle rewrite). A salvaged copy preserves local seqnos and holds
1723 /// the source's content, so the position applies to it unchanged, and its
1724 /// presence also says the offset need not be reconstructed from the SST,
1725 /// so the fail-closed bulk-ingest rejection does not apply.
1726 ///
1727 /// A zero offset is genuine, NOT a sentinel: the first ingestion on a
1728 /// fresh counter commits offset 0. Only `None` means "unknown".
1729 recovered_position: Option<ManifestPosition>,
1730}
1731
1732fn try_salvage_table(
1733 config: &Config,
1734 fs: &Arc<dyn crate::fs::Fs>,
1735 allow_resurrection: bool,
1736 salvage: TableSalvage<'_>,
1737) -> crate::Result<SalvageOutcome> {
1738 let TableSalvage {
1739 source,
1740 table_path,
1741 table_id,
1742 reject_punched_without_bound,
1743 blob_rewrite,
1744 owner_promotions,
1745 recovered_position,
1746 } = salvage;
1747 // Salvage under the tree's configured comparator + crypto/dictionary context
1748 // so the rewritten SST opens, orders, and decrypts / decompresses consistently
1749 // with the rest of the tree on reopen (the reopen below uses the same
1750 // `config.encryption` and the same dictionary set).
1751 let report = crate::salvage::salvage_with_context(
1752 source,
1753 table_path.to_path_buf(),
1754 fs,
1755 &config.comparator,
1756 &crate::salvage::SalvageOptions {
1757 encryption: config.encryption.clone(),
1758 #[cfg(zstd_any)]
1759 zstd_dictionary: config.zstd_dictionary.clone(),
1760 // The whole set, not just the configured one: the source may have
1761 // been written under a dictionary the tree stores and the caller
1762 // never supplied, and salvage cannot read a block it cannot resolve.
1763 #[cfg(zstd_any)]
1764 zstd_dictionaries: config.current_zstd_dictionaries(),
1765 // The real table id, so encrypted block AAD (which binds it) decrypts
1766 // and the recovered copy reopens under the same id below.
1767 table_id,
1768 // Repair KNOWS the durable id (the file name), so the salvage
1769 // open cross-checks the meta payload against it: a forged tail
1770 // id falls back to the intact MID mirror instead of stamping the
1771 // recovered copy with an identity the reopen below would reject.
1772 expected_stored_id: Some(table_id),
1773 // Governed by the recovery-wide resurrection flag: off (default), a
1774 // delete-bearing SST whose bitmap cannot be authenticated is excluded
1775 // rather than masked against an unverified bitmap; on, its rows are
1776 // re-emitted live, accepting that deleted rows reappear.
1777 allow_delete_resurrection: allow_resurrection,
1778 // The recovered SST is persisted at the tree's configured
1779 // durability, matching the manifest rebuilt around it.
1780 sync_mode: config.sync_mode,
1781 // The extractor is configuration, not persisted state: without
1782 // it the rebuilt filter loses the source's prefix hashes and
1783 // prefix scans see the salvaged copy as definitely absent.
1784 prefix_extractor: config.prefix_extractor.clone(),
1785 blob_rewrite,
1786 owner_promotions,
1787 // A repair replacement always keeps the SOURCE's identity: it is
1788 // built at `{id}.repair-tmp` and swapped onto `{id}`, so the name it
1789 // ends up under is the identity it was stamped with.
1790 output_id: None,
1791 // Forward the caller's live-progress handle so the block walk
1792 // ticks per inspected / recovered block while it runs.
1793 progress: config.recovery_progress.clone(),
1794 // The copy is stamped on the tree's clock, which FIFO judges it on.
1795 clock: config.clock.clone(),
1796 },
1797 )?;
1798 if report.salvaged_path.is_none() {
1799 return Ok(SalvageOutcome::Unusable);
1800 }
1801 if !report.dropped.is_empty() {
1802 log::warn!(
1803 "salvaged table {table_id}: recovered {} block(s), dropped {} corrupt block(s)",
1804 report.blocks_salvaged,
1805 report.dropped.len(),
1806 );
1807 }
1808 if reject_punched_without_bound
1809 && dropped_data_extent_is_zeroed(&**fs, source, &report.dropped)?
1810 {
1811 // The source was punched but its bound is unrecoverable: the salvaged
1812 // replacement re-emits every intact block — including consumed,
1813 // superseded blocks a partial punch left inside the reclaimed prefix —
1814 // with nothing to restrict them. Reject it: this copy is our own
1815 // byproduct, holds nothing the untouched source does not, and no
1816 // manifest will ever name it, so it is removed outright. A removal
1817 // failure PROPAGATES — left behind, the numeric SST is an orphan the
1818 // next open must delete, and that open would fail on the same error.
1819 discard_unreferenced(&**fs, table_path, config.sync_mode)?;
1820 return Ok(SalvageOutcome::PunchedBoundLost);
1821 }
1822
1823 // Reopen the freshly-written (clean) salvaged SST so it joins the rebuilt
1824 // manifest like any cleanly-recovered table. A clean manifest record's
1825 // ingest offset applies to the copy unchanged (the salvage preserves
1826 // local seqnos), so it is reused here just like on the whole-recover path.
1827 let checksum = crate::Checksum::from_raw(compute_table_checksum(&**fs, table_path)?);
1828 let table = match Table::recover(held_recover_params(
1829 config,
1830 table_path.to_path_buf(),
1831 checksum,
1832 table_id,
1833 Arc::clone(fs),
1834 recovered_position,
1835 )) {
1836 Ok(table) => table,
1837 Err(e) => {
1838 // Same rule as the rejection below: the copy is this pass's own
1839 // byproduct and no manifest will ever name it, so leaving it makes
1840 // an orphan the next open must delete — and a retry writes another
1841 // one beside it. Remove it; the reopen failure is what the caller
1842 // sees.
1843 if let Err(rm) = discard_unreferenced(&**fs, table_path, config.sync_mode) {
1844 log::error!(
1845 "salvaged copy {} could not be removed after its reopen failed ({rm}); \
1846 it stays until the next orphan sweep",
1847 table_path.display(),
1848 );
1849 }
1850 return Err(e);
1851 }
1852 };
1853 // A salvaged copy of a bulk-ingested source still relies on the manifest-only
1854 // global_seqno offset a manifest-LOSS rebuild cannot recover: its entries stay
1855 // at local seqno 0, so installing it with offset 0 would silently mis-order
1856 // and over-expose them. Without a recovered offset to reuse, treat it as
1857 // unsalvageable — remove the freshly-written copy and let the caller record
1858 // the table unreadable.
1859 if recovered_position.is_none()
1860 && has_unrecoverable_ingest_offset(
1861 table.metadata.bulk_ingested,
1862 table.metadata.item_count,
1863 table.max_local_seqno(),
1864 )
1865 {
1866 release_held(config, table);
1867 // Remove the rejected replacement, and do NOT swallow the error. A
1868 // discarded `remove_file` failure would leave the freshly-written
1869 // numeric SST in `tables/`; repair would still install a manifest that
1870 // omits it and report success, but the next open classifies it as an
1871 // orphan and fails on the SAME persistent deletion, so the "repaired"
1872 // tree cannot reopen. Propagating instead keeps the two outcomes intact.
1873 discard_unreferenced(&**fs, table_path, config.sync_mode)?;
1874 return Ok(SalvageOutcome::Unusable);
1875 }
1876 Ok(SalvageOutcome::Salvaged(table))
1877}
1878
1879/// Whether any salvage-dropped DATA extent of `source` contains a
1880/// structure-anchored all-zero run — the hole-punch signature. A real data
1881/// block's bytes are never all zero across a whole block, so a header-length
1882/// run that ends where intact structure begins (a decodable block header, the
1883/// next dropped extent, or the data-section end) was physically reclaimed, not
1884/// merely corrupted. Restricting the scan to the DROPPED extents keeps
1885/// legitimate zero runs inside intact (checksum-clean) blocks from
1886/// false-positiving, and the structural anchor keeps header-sized zero runs
1887/// inside a damaged extent's VALUE payloads from doing the same.
1888///
1889/// The scan covers each dropped extent IN FULL, up to the next dropped extent
1890/// or the end of the `data` section, not just its opening window: when the
1891/// physical chain breaks, the salvage walk surrenders the whole remaining tail
1892/// as ONE extent whose offset is the first DAMAGED (nonzero) frame, so punched
1893/// blocks deeper inside it would otherwise stay invisible and the salvaged
1894/// output would publish consumed records unrestricted.
1895///
1896/// # Errors
1897///
1898/// Propagates the open / read failure (a transient one aborts the repair for a
1899/// retry, exactly like the other salvage-path reads).
1900#[cfg(feature = "std")]
1901fn dropped_data_extent_is_zeroed(
1902 fs: &dyn crate::fs::Fs,
1903 source: &std::path::Path,
1904 dropped: &[crate::salvage::DroppedBlock],
1905) -> crate::Result<bool> {
1906 let scan = excised_extents(fs, source, dropped)?;
1907 // An UNATTRIBUTABLE qualifying run counts too: on a punch-capable mount
1908 // a zero run the hole probe cannot answer for is indistinguishable from
1909 // a lost-sidecar reclaim, and the caller's fail-closed guard (active
1910 // only without resurrection) must fire rather than publish the salvaged
1911 // copy unrestricted.
1912 Ok(!scan.proven.is_empty() || scan.unattributed)
1913}
1914
1915/// Result of [`excised_extents`]: the zero runs proven to be reclaimed, and
1916/// whether any qualifying run could not be attributed either way.
1917#[cfg(feature = "std")]
1918struct ExcisedScan {
1919 /// Structure-anchored zero runs proven to carry a physical hole,
1920 /// ascending and disjoint.
1921 proven: Vec<(u64, u64)>,
1922 /// Whether a qualifying zero run on a PUNCH-CAPABLE mount got `None`
1923 /// from the hole probe — reclaim and damage indistinguishable.
1924 unattributed: bool,
1925}
1926
1927/// The physically excised byte ranges of `source`'s data section: the
1928/// structure-anchored all-zero runs inside its dropped extents, which is what
1929/// a hole punch leaves behind. Derived purely by scanning, so the result
1930/// survives any crash without being recorded anywhere — the property the
1931/// recovery model depends on for in-place repair (see
1932/// `docs/manifest-recovery.md`).
1933///
1934/// Ranges come back ascending and disjoint. A PREFIX punch yields one range
1935/// starting at the data section; a mid-file punch yields an interior range,
1936/// which the prefix-only restriction model cannot express — see the weak-spot
1937/// list in the same document.
1938///
1939/// # Errors
1940///
1941/// Propagates the open / read failure (a transient one aborts the repair for a
1942/// retry, exactly like the other salvage-path reads).
1943#[cfg(feature = "std")]
1944fn excised_extents(
1945 fs: &dyn crate::fs::Fs,
1946 source: &std::path::Path,
1947 dropped: &[crate::salvage::DroppedBlock],
1948) -> crate::Result<ExcisedScan> {
1949 // Shortest run accepted as a punch. A hole is punched per DATA BLOCK, so a
1950 // punched block contributes a zero run at least a block long — while inside
1951 // an intact block a zero run is bounded by its framing (header, key/value
1952 // lengths and checksums are never all zero across a whole block). The
1953 // block-header length is the smallest possible block, so a run of that many
1954 // zeros cannot come from one intact framed block.
1955 const MIN_RUN: u64 = crate::table::block::Header::MIN_LEN as u64;
1956 let mut excised: Vec<(u64, u64)> = Vec::new();
1957 if dropped.is_empty() {
1958 return Ok(ExcisedScan {
1959 proven: excised,
1960 unattributed: false,
1961 });
1962 }
1963 let mut file = fs.open(source, &crate::fs::FsOpenOptions::new().read(true))?;
1964 let file_len = crate::fs::FsFile::metadata(&*file)?.len;
1965 // The `data` section's physical end bounds every extent: a dropped extent
1966 // runs to the next dropped one or to that end, whichever comes first.
1967 let data_end = match crate::sfa::Reader::from_reader(&mut file) {
1968 Ok(reader) => reader
1969 .toc()
1970 .iter()
1971 .find(|e| e.name() == b"data")
1972 .map_or(file_len, |e| e.pos().saturating_add(e.len()).min(file_len)),
1973 // No readable TOC (the very corruption salvage is recovering from):
1974 // fall back to the file end — a superset of the data section, and the
1975 // per-extent scan below is bounded by the next extent anyway.
1976 Err(_) => file_len,
1977 };
1978 let mut starts: Vec<u64> = dropped
1979 .iter()
1980 .filter(|d| d.section == b"data" && d.offset < data_end)
1981 .map(|d| d.offset)
1982 .collect();
1983 starts.sort_unstable();
1984 starts.dedup();
1985
1986 // A qualifying run must additionally be STRUCTURE-ANCHORED: it counts as
1987 // punch evidence only when it ends where intact structure begins — a
1988 // decodable block header (magic + type + the header's own checksum), the
1989 // next dropped extent, or the data-section end. SST values are arbitrary
1990 // bytes, so a header-sized zero run INSIDE a damaged extent's payload is
1991 // otherwise indistinguishable from a punch by length alone, and a bare
1992 // length test would reject an otherwise usable salvage as bound-lost under
1993 // the default no-resurrection policy.
1994 let header_decodes_at = |pos: u64| -> crate::Result<bool> {
1995 use crate::coding::Decode;
1996 let max = crate::table::block::Header::MAX_LEN as u64;
1997 let want = usize::try_from(file_len.saturating_sub(pos).min(max)).unwrap_or(0);
1998 if want < crate::table::block::Header::MIN_LEN {
1999 return Ok(false);
2000 }
2001 let bytes = crate::file::read_exact(&*file, pos, want)?;
2002 Ok(crate::table::block::Header::decode_from(&mut &bytes[..]).is_ok())
2003 };
2004
2005 const CHUNK: usize = 64 * 1024;
2006 for (i, &start) in starts.iter().enumerate() {
2007 let end = starts.get(i + 1).copied().unwrap_or(data_end).min(data_end);
2008 let mut offset = start;
2009 let mut run: u64 = 0;
2010 while offset < end {
2011 let want = usize::try_from(end - offset).unwrap_or(CHUNK).min(CHUNK);
2012 let bytes = crate::file::read_exact(&*file, offset, want)?;
2013 for (j, &b) in bytes.iter().enumerate() {
2014 if b == 0 {
2015 run += 1;
2016 } else {
2017 let run_end = offset + j as u64;
2018 if run >= MIN_RUN && header_decodes_at(run_end)? {
2019 excised.push((run_end - run, run_end));
2020 }
2021 run = 0;
2022 }
2023 }
2024 offset += want as u64;
2025 }
2026 // A run reaching the extent end needs no header anchor: it terminates
2027 // at the next dropped extent or the data-section end, both of which
2028 // are structural boundaries themselves.
2029 if run >= MIN_RUN {
2030 excised.push((end - run, end));
2031 }
2032 }
2033 // The per-extent walks are already ascending and cannot overlap (each is
2034 // bounded by the next extent), but a run that ends exactly where the next
2035 // begins is one hole physically — merge so callers see maximal ranges.
2036 excised.sort_unstable();
2037 let mut merged: Vec<(u64, u64)> = Vec::with_capacity(excised.len());
2038 for (start, end) in excised {
2039 match merged.last_mut() {
2040 Some(last) if start <= last.1 => last.1 = last.1.max(end),
2041 _ => merged.push((start, end)),
2042 }
2043 }
2044 // Zeros are the SHAPE of a reclaim, not the proof: corruption that destroys
2045 // a data block leaves the same read-as-zeros run, and calling that a punch
2046 // condemns an otherwise salvageable table as bound-lost. A reclaim
2047 // deallocates, so each run must cover an actual HOLE. A run proven
2048 // ALLOCATED (`Some(false)`) is damage; a run the backend CANNOT attribute
2049 // (`None`) on a punch-capable mount is reported separately — reclaim and
2050 // damage are then indistinguishable, and the caller's no-resurrection
2051 // guard must fail closed rather than read "unproven" as "unpunched".
2052 //
2053 // Probed as CONTAINS-a-hole over the run rather than whole-span-is-a-hole
2054 // or a single midpoint byte: the reclaim punches at unaligned extents (an
2055 // older reclaim did so per block), so the filesystem deallocates only the
2056 // wholly-contained pages and leaves zero-filled boundary pages allocated:
2057 // the whole span is never a hole, and a midpoint byte can land on one of
2058 // those allocated boundary pages. Any hole inside the zeroed run is attributable to it.
2059 // Only the presence of the hole matters here — the single caller asks
2060 // whether the extent was reclaimed at all, not exactly where.
2061 let mut proven = Vec::with_capacity(merged.len());
2062 let mut unattributed = false;
2063 for (start, end) in merged {
2064 match fs.extent_contains_hole(source, start, end - start)? {
2065 Some(true) => proven.push((start, end)),
2066 Some(false) => {}
2067 None => unattributed = true,
2068 }
2069 }
2070 if unattributed {
2071 unattributed = fs.capabilities(source).punch_hole;
2072 }
2073 Ok(ExcisedScan {
2074 proven,
2075 unattributed,
2076 })
2077}
2078
2079/// Whether the source SST's first data block reads as all zeros, the signature of
2080/// a hole-punched prefix. Probed at offset 0 (the `data` section is written first)
2081/// over a small window that stays WITHIN the first block, so even a punch of only
2082/// the first block is detected; a real data block's opening bytes (its first
2083/// entry's key-length varint and key) are never all zero, so an unpunched SST can
2084/// never false-positive.
2085///
2086/// Used by the recovery-failure salvage arm to fail closed on a PUNCHED source
2087/// whose bound is unrecoverable (missing / corrupt sidecar and no `Table` to
2088/// derive from): salvaging it into an unrestricted output would resurrect the
2089/// straddling block's sub-bound rows.
2090///
2091/// This is the CHEAP pre-salvage fast path only: a PARTIAL punch (the
2092/// punch-on-drop reclaim continues past an individual `punch_hole` failure) can
2093/// leave the first block intact while later prefix blocks are zeroed, which this
2094/// probe cannot see. [`dropped_data_extent_is_zeroed`] closes that gap after the
2095/// salvage walk, whose dropped extents expose every zeroed block.
2096///
2097/// # Errors
2098///
2099/// Propagates the open / read failure.
2100#[cfg(feature = "std")]
2101fn source_prefix_is_punched(
2102 fs: &dyn crate::fs::Fs,
2103 table_path: &std::path::Path,
2104) -> crate::Result<bool> {
2105 const PROBE: usize = 64;
2106 let file = fs.open(table_path, &crate::fs::FsOpenOptions::new().read(true))?;
2107 let len = crate::fs::FsFile::metadata(&*file)?.len;
2108 if len == 0 {
2109 return Ok(false);
2110 }
2111 let n = usize::try_from(len).unwrap_or(PROBE).min(PROBE);
2112 let bytes = crate::file::read_exact(&*file, 0, n)?;
2113 if !bytes.iter().all(|&b| b == 0) {
2114 return Ok(false);
2115 }
2116 // Zeros are not the evidence — corruption produces them too, and reading
2117 // ordinary damage as a lost punch bound sets the whole table aside instead
2118 // of salvaging the blocks that are still readable. A reclaim deallocates,
2119 // so the probed window must carry a HOLE — contains-a-hole, the same rule
2120 // the block-level punch classifier applies (an unaligned punch keeps its
2121 // zero-filled edges allocated).
2122 //
2123 // A backend that cannot answer (`None`) reports NOT-punched here, which is
2124 // deliberately the permissive direction: this is only the cheap fast path,
2125 // and answering "punched" on an unproven window would set aside every
2126 // ordinarily-damaged table on a mount without extent reporting. The
2127 // fail-closed decision for unattributable zeros belongs to
2128 // `dropped_data_extent_is_zeroed` after the salvage walk, which sees every
2129 // dropped all-zero extent rather than just this window and reports them
2130 // (see `ExcisedScan::unattributed`).
2131 Ok(fs.extent_contains_hole(table_path, 0, n as u64)? == Some(true))
2132}
2133
2134/// Re-imposes a tight-space restriction on a SALVAGED replacement SST, the single
2135/// point every salvage output funnels through so the restriction can never be
2136/// dropped on one path and kept on another.
2137///
2138/// Salvage rewrites its source as a fresh, UNPUNCHED table that re-emits the
2139/// straddling block's sub-bound rows, so a punched source's restriction must be
2140/// re-applied to the output or those superseded / deleted rows resurrect. With a
2141/// known `bound` and resurrection off, the sidecar is re-written (so a later
2142/// manifest-loss repair honors it against the now-unpunched file) and the table
2143/// reopened restricted. Otherwise (resurrection on, or no recoverable bound) the
2144/// salvaged table is kept whole and any stale sidecar cleared, since a lingering
2145/// sidecar would wrongly restrict the unpunched replacement on a later repair.
2146///
2147/// ANY failure re-imposing the restriction (sidecar write or restricted reopen)
2148/// removes the half-finished replacement before propagating. It carries the
2149/// straddling block's sub-bound rows with no valid sidecar, so a run that found
2150/// it would adopt it UNRESTRICTED and resurrect exactly the rows the restriction
2151/// hid. The source is untouched either way, so the retry re-salvages and
2152/// re-restricts from it. This holds for a PERSISTENT failure (an ENOSPC on the
2153/// sidecar write) as much as a transient one.
2154#[cfg(feature = "std")]
2155fn restrict_salvaged_output(
2156 folder_fs: &dyn crate::fs::Fs,
2157 config: &Config,
2158 table_path: &std::path::Path,
2159 salvaged: Table,
2160 restrict_bound: Option<crate::UserKey>,
2161 allow_resurrection: bool,
2162) -> crate::Result<Table> {
2163 match restrict_bound {
2164 Some(bound) if !allow_resurrection => {
2165 let table_id = salvaged.metadata.id;
2166 let restricted = crate::restrict_bound::write(
2167 folder_fs,
2168 table_path,
2169 config.encryption.as_deref(),
2170 table_id,
2171 &bound,
2172 config.sync_mode,
2173 )
2174 .and_then(|()| salvaged.reopen_restricted_with(bound, held_descriptors(config)));
2175 match restricted {
2176 Ok(table) => Ok(table),
2177 Err(e) => {
2178 // Drop the salvaged handle's open file BEFORE removing it: a
2179 // backend that refuses to unlink an OPEN file (Windows; the
2180 // deletion path closes handles for this same reason) would
2181 // fail while `salvaged` still holds it.
2182 release_held(config, salvaged);
2183 // Remove on EVERY failure, transient or persistent: the
2184 // replacement is unpunched with no valid sidecar, so a run
2185 // that adopted it would resurrect the sub-bound rows. The
2186 // source is untouched, so the retry re-salvages from it. A
2187 // removal that itself fails propagates THAT error — an
2188 // undiscardable half-finished replacement is the one thing
2189 // that could corrupt the retry.
2190 discard_unreferenced(folder_fs, table_path, config.sync_mode)?;
2191 Err(e)
2192 }
2193 }
2194 }
2195 _ => {
2196 crate::restrict_bound::remove(folder_fs, table_path, config.sync_mode);
2197 Ok(salvaged)
2198 }
2199 }
2200}
2201
2202/// The scanned totals of a validated blob file's LIVE frames. For a punched
2203/// file these cover only the suffix, and the difference to the whole-file
2204/// metadata totals is exactly the punched prefix's garbage — what seeds the
2205/// rebuilt manifest's fragmentation accounting.
2206#[cfg(feature = "std")]
2207struct BlobLiveTotals {
2208 items: u64,
2209 uncompressed_bytes: u64,
2210 compressed_bytes: u64,
2211}
2212
2213/// What a blob file's punch-geometry walk proved about its live data.
2214#[cfg(feature = "std")]
2215#[derive(Debug)]
2216enum BlobFrontier {
2217 /// No punched prefix: the whole file is live.
2218 Whole,
2219 /// Live data begins at this byte offset; the `[data_start, offset)` prefix
2220 /// is punched.
2221 Punched(u64),
2222 /// The punch consumed EVERY frame: the relocation completed and only the
2223 /// file's removal lagged the crash. No live data remains.
2224 FullyConsumed,
2225}
2226
2227/// Derives a blob file's tight-space live-data frontier from its on-disk punch
2228/// geometry, for a manifest-loss repair.
2229///
2230/// The frontier — where a tight-space relocation's punched `[data_start,
2231/// frontier)` prefix ends and the live suffix begins — is recorded only in the
2232/// manifest's `blob_restrictions` section. Unlike an SST's restriction bound (a
2233/// KEY, which the block-aligned punch cannot reproduce and which therefore
2234/// needs its `.restrict-bound` sidecar), the blob frontier is a byte offset at
2235/// a frame boundary, so the geometry recovers it EXACTLY: the punch zeroes
2236/// precisely `[data_start, frontier)` and the first live frame's magic sits at
2237/// `frontier`.
2238///
2239/// Anchoring is structural, never length-based: a zeroed run counts only when a
2240/// frame decodes cleanly at its end, so a zero-filled value payload inside the
2241/// live suffix (stepped over by frame framing) can never move the frontier. A
2242/// partially completed punch (a crash mid-reclaim can leave intact-but-consumed
2243/// frames between holes) is walked hole-by-hole, and the frontier is the end of
2244/// the LAST zeroed run the anchored walk reaches. Non-zero bytes that fail to
2245/// decode end the walk at the last anchored frontier: content corruption is not
2246/// punch geometry, and it surfaces exactly as it would on an unpunched file.
2247///
2248/// Returns [`BlobFrontier::Whole`] when the first data byte is non-zero: the
2249/// punch always starts at the data start, so an unpunched file — including one
2250/// whose committed punch never ran before a crash — short-circuits without a
2251/// walk, keeping the common repair path at zero extra read cost. The redundant
2252/// unpunched prefix is superseded by relocated copies and reclaimed later, the
2253/// same safe fallback the SST path takes for a committed-but-unpunched slice.
2254/// Zeros through the whole data section are [`BlobFrontier::FullyConsumed`]:
2255/// a completed relocation whose file removal lagged the crash.
2256///
2257/// # Errors
2258///
2259/// Propagates I/O and TOC errors (the caller classifies transient ones for
2260/// retry, like every other per-file repair probe).
2261fn derive_blob_frontier(
2262 fs: &Arc<dyn crate::fs::Fs>,
2263 path: &std::path::Path,
2264 blob_id: crate::vlog::BlobFileId,
2265) -> crate::Result<BlobFrontier> {
2266 let mut file = fs.open(path, &crate::fs::FsOpenOptions::new().read(true))?;
2267 let (data_start, data_end) = {
2268 let reader = crate::sfa::Reader::from_reader(&mut file)?;
2269 let data = reader
2270 .toc()
2271 .section(b"data")
2272 .ok_or(crate::Error::InvalidHeader("BlobFile"))?;
2273 let end = data
2274 .pos()
2275 .checked_add(data.len())
2276 .ok_or(crate::Error::InvalidHeader("BlobFile"))?;
2277 (data.pos(), end)
2278 };
2279 if data_start >= data_end {
2280 return Ok(BlobFrontier::Whole);
2281 }
2282
2283 // Ends of the contiguous all-zero run starting at `from` (chunked reads,
2284 // capped by the data-section end).
2285 let skip_zeros = |from: u64| -> crate::Result<u64> {
2286 const CHUNK: u64 = 64 * 1024;
2287 let mut pos = from;
2288 while pos < data_end {
2289 // `#[allow]`, not `#[expect]`: target-width-dependent lint (`u64 as
2290 // usize`) — on 64-bit targets Clippy proves the `min()` bound fits
2291 // usize and an `#[expect]` would be unfulfilled under `-D warnings`.
2292 #[allow(
2293 clippy::cast_possible_truncation,
2294 reason = "min() bounds the window by CHUNK, which fits usize"
2295 )]
2296 let want = (data_end - pos).min(CHUNK) as usize;
2297 let chunk = crate::file::read_exact(&*file, pos, want)?;
2298 match chunk.iter().position(|b| *b != 0) {
2299 Some(hit) => return Ok(pos + hit as u64),
2300 None => pos += want as u64,
2301 }
2302 }
2303 Ok(data_end)
2304 };
2305
2306 // Whether a zero run was physically RECLAIMED rather than merely zeroed.
2307 // Zeros are not the evidence: ordinary corruption produces exactly the same
2308 // bytes, and reading damage as geometry drops every handle below a
2309 // fabricated frontier — or, for a wholly zeroed section, removes a file
2310 // whose records were only damaged. A punch deallocates, so the run must
2311 // read back as a hole. The same rule the SST classifiers apply.
2312 //
2313 // Probed as CONTAINS-a-hole over the run: the run is bounded by the
2314 // neighbouring non-zero bytes (so demanding the whole span be unallocated
2315 // would reject genuine punches), and the punch's zero-filled edge pages
2316 // stay allocated on a real filesystem (so a single midpoint byte can land
2317 // on an allocated page even inside a genuine reclaim). A backend that
2318 // cannot answer leaves the run unproven, which keeps the zeros classified
2319 // as damage.
2320 let is_reclaimed = |from: u64, to: u64| -> crate::Result<bool> {
2321 if to <= from {
2322 return Ok(false);
2323 }
2324 Ok(fs.extent_contains_hole(path, from, to - from)? == Some(true))
2325 };
2326
2327 // Fast path: an unpunched file's first frame magic (non-zero) sits at the
2328 // data start.
2329 if skip_zeros(data_start)? == data_start {
2330 return Ok(BlobFrontier::Whole);
2331 }
2332
2333 // The last structure-anchored frontier: committed only once a frame has
2334 // decoded cleanly at a zeroed run's end. Zero = no run was ever anchored,
2335 // which reports the file whole (an unproven punch is content, not
2336 // geometry).
2337 let committed = |c: u64| {
2338 if c == 0 {
2339 BlobFrontier::Whole
2340 } else {
2341 BlobFrontier::Punched(c)
2342 }
2343 };
2344 let mut pos = data_start;
2345 let mut anchored: u64 = 0;
2346 loop {
2347 let run_end = skip_zeros(pos)?;
2348 // An UNRECLAIMED run is destroyed content, not geometry: stop at the
2349 // last proven frontier and let validation surface the damage exactly as
2350 // it would on an unpunched file.
2351 if !is_reclaimed(pos, run_end)? {
2352 return Ok(committed(anchored));
2353 }
2354 if run_end >= data_end {
2355 // Zeros to the section end. Only a completed relocation when
2356 // NOTHING was anchored below them: reclaim punches the consumed
2357 // prefix top-down from the data start, so a zeroed tail that
2358 // FOLLOWS intact anchored frames cannot be punch geometry — it is
2359 // destroyed data. Reporting it as consumed would drop a file whose
2360 // live frames are still referenced (and every table pointing at
2361 // it); keeping the anchored frontier instead lets the damaged
2362 // suffix surface through validation, exactly as the same damage
2363 // would on an unpunched file.
2364 return Ok(if anchored == 0 {
2365 BlobFrontier::FullyConsumed
2366 } else {
2367 committed(anchored)
2368 });
2369 }
2370 let mut scanner = crate::vlog::BlobFileScanner::resume(path, &**fs, blob_id, run_end)?;
2371 match scanner.next() {
2372 Some(Ok(entry)) if !entry.resynced => {
2373 anchored = run_end;
2374 pos = entry.frame_end;
2375 }
2376 Some(Err(e)) if is_environmental(&e) => return Err(e),
2377 // The zeroed run is not punch geometry (no frame decodes at its
2378 // end): keep the last anchored frontier.
2379 _ => return Ok(committed(anchored)),
2380 }
2381 // Chain frames from the anchor until the section ends cleanly or the
2382 // chain breaks (another hole, or content corruption).
2383 loop {
2384 match scanner.next() {
2385 None => return Ok(committed(anchored)),
2386 Some(Ok(entry)) if !entry.resynced => pos = entry.frame_end,
2387 Some(Err(e)) if is_environmental(&e) => return Err(e),
2388 Some(Ok(_) | Err(_)) => {
2389 // The frame starting at `pos` failed (or the scanner
2390 // resynced past unproven bytes). Another zeroed hole
2391 // continues the walk; anything else is content corruption
2392 // and ends it at the last anchored frontier.
2393 if skip_zeros(pos)? == pos {
2394 return Ok(committed(anchored));
2395 }
2396 break;
2397 }
2398 }
2399 }
2400 }
2401}
2402
2403/// Records a recovered table the blob-dependency stage cannot publish: reported
2404/// unreadable, and removed once the manifest is durable. Consumes the handle.
2405///
2406/// The file is NOT touched here. This stage runs before the commit, where the
2407/// source is still the only copy of its rows and a crash must leave the
2408/// directory exactly as a retry expects to find it.
2409#[cfg(feature = "std")]
2410fn set_aside_table(
2411 table: Table,
2412 reason: &str,
2413 unreadable_files: &mut Vec<(PathBuf, String)>,
2414 discard_after_commit: &mut Vec<(Arc<dyn crate::fs::Fs>, PathBuf, String)>,
2415) {
2416 let path = (*table.path).clone();
2417 let fs = table.fs.clone();
2418 drop(table); // release the handle
2419 discard_after_commit.push((fs, path.clone(), reason.to_string()));
2420 unreadable_files.push((path, reason.to_string()));
2421}
2422
2423/// Same as [`set_aside_table`] for a source that is no longer open.
2424#[cfg(feature = "std")]
2425fn set_aside_path(
2426 fs: &Arc<dyn crate::fs::Fs>,
2427 path: &std::path::Path,
2428 reason: &str,
2429 unreadable_files: &mut Vec<(PathBuf, String)>,
2430 discard_after_commit: &mut Vec<(Arc<dyn crate::fs::Fs>, PathBuf, String)>,
2431) {
2432 discard_after_commit.push((Arc::clone(fs), path.to_path_buf(), reason.to_string()));
2433 unreadable_files.push((path.to_path_buf(), reason.to_string()));
2434}
2435
2436/// Whether every frame in `path`'s live data range (`[live_data_start, end)`)
2437/// verifies. Repair must not record a digest over damaged content: the
2438/// restamped digest would launder the corruption past every later integrity
2439/// check while reads of the affected values still fail — such a file is
2440/// salvaged instead. Framing checks alone are not enough, because the frame
2441/// checksum is unkeyed and covers only the ON-DISK bytes; each acceptance
2442/// criterion below closes a distinct restamp/reorder shape:
2443///
2444/// - every frame decodes and checksums cleanly, with no resynchronization;
2445/// - a compressed frame's payload DECOMPRESSES (a re-stamped checksum over an
2446/// undecodable compressed payload frames cleanly, yet every live read of
2447/// the value fails);
2448/// - frame keys never regress under the tree comparator (individually-valid
2449/// frames reordered on disk break the sorted-input contract every blob
2450/// reader and the relocation merge scanner rely on);
2451/// - for an unpunched file, the metadata counters match the scanned frames
2452/// (the meta block's item count, uncompressed byte total, and key range are
2453/// what blob GC's dead-file arithmetic trusts — an understated total lets
2454/// `is_dead` reclaim a file whose uncounted frames are still referenced).
2455/// A punched file's metadata describes the whole original file while the
2456/// scan covers only the live suffix, so it is checked against the LOWER
2457/// bounds the subset relation implies instead: totals at least the suffix
2458/// totals, key range containing the scanned suffix.
2459///
2460/// Returns the scanned live totals on success — the caller seeds the punched
2461/// prefix's garbage accounting from their difference to the whole-file
2462/// metadata.
2463///
2464/// # Errors
2465///
2466/// Propagates transient I/O for retry; any structural or persistent frame
2467/// failure is a conclusive `Ok(None)`.
2468#[cfg(feature = "std")]
2469fn validate_blob_frames(
2470 config: &Config,
2471 path: &std::path::Path,
2472 blob_id: crate::vlog::BlobFileId,
2473 live_data_start: u64,
2474 // The handle the caller already opened for the identity check: its meta
2475 // section carries both the compression descriptor and the recorded
2476 // totals this cross-check needs, so re-opening it here read the same
2477 // section twice per file.
2478 handle: &crate::vlog::BlobFile,
2479 // What `handle`'s descriptor names (see `blob_file_dictionary`), resolved
2480 // by the caller, which alone knows whether a missing one stops the repair.
2481 #[cfg(zstd_any)] dict: Option<&Arc<crate::compression::ZstdDictionary>>,
2482) -> crate::Result<Option<BlobLiveTotals>> {
2483 let fs = &config.fs;
2484 let compression = handle.compression();
2485 let comparator = &config.comparator;
2486
2487 let scanner = if live_data_start > 0 {
2488 crate::vlog::BlobFileScanner::resume(path, &**fs, blob_id, live_data_start)?
2489 } else {
2490 crate::vlog::BlobFileScanner::new(path, &**fs, blob_id)?
2491 };
2492 let mut count: u64 = 0;
2493 let mut uncompressed_total: u64 = 0;
2494 let mut compressed_total: u64 = 0;
2495 let mut first_key: Option<crate::UserKey> = None;
2496 let mut prev: Option<(crate::UserKey, crate::SeqNo)> = None;
2497 for item in scanner {
2498 match item {
2499 Ok(entry) if !entry.resynced => {
2500 if crate::salvage::blob_key_regresses(comparator, prev.as_ref(), &entry) {
2501 log::warn!(
2502 "blob file {blob_id} at {}: frame at {} regresses below the \
2503 previous frame's key — the frames were reordered",
2504 path.display(),
2505 entry.offset,
2506 );
2507 return Ok(None);
2508 }
2509 if crate::salvage::decompress_blob_value(
2510 compression,
2511 &entry.value,
2512 entry.uncompressed_len as usize,
2513 #[cfg(zstd_any)]
2514 dict.map(Arc::as_ref),
2515 )
2516 .is_err()
2517 {
2518 log::warn!(
2519 "blob file {blob_id} at {}: frame at {} does not decompress \
2520 despite a clean checksum",
2521 path.display(),
2522 entry.offset,
2523 );
2524 return Ok(None);
2525 }
2526 count += 1;
2527 uncompressed_total += u64::from(entry.uncompressed_len);
2528 // Sum of u32-bounded on-disk lengths within one file: cannot
2529 // overflow u64 (the file itself cannot reach 2^64 bytes).
2530 compressed_total += entry.value.len() as u64;
2531 if first_key.is_none() {
2532 first_key = Some(entry.key.clone());
2533 }
2534 prev = Some((entry.key.clone(), entry.seqno));
2535 }
2536 Err(e) if is_environmental(&e) => return Err(e),
2537 // A resynced frame has an unprovable boundary (damage upstream);
2538 // any other error is a structural or persistent frame failure.
2539 // Both are conclusive: this file's frames do not all verify.
2540 Ok(_) | Err(_) => return Ok(None),
2541 }
2542 }
2543
2544 let meta = handle.meta();
2545 if live_data_start == 0 {
2546 let range_matches = match (&first_key, &prev) {
2547 (Some(first), Some((last, _))) => {
2548 meta.key_range.min().as_ref() == first.as_ref()
2549 && meta.key_range.max().as_ref() == last.as_ref()
2550 }
2551 _ => count == 0,
2552 };
2553 if meta.item_count != count
2554 || meta.total_uncompressed_bytes != uncompressed_total
2555 || meta.total_compressed_bytes != compressed_total
2556 || !range_matches
2557 {
2558 log::warn!(
2559 "blob file {blob_id} at {}: metadata disagrees with the scanned frames \
2560 (meta: {} items / {} uncompressed bytes; scanned: {count} / \
2561 {uncompressed_total})",
2562 path.display(),
2563 meta.item_count,
2564 meta.total_uncompressed_bytes,
2565 );
2566 return Ok(None);
2567 }
2568 } else {
2569 // A punched file's metadata describes the WHOLE original file while
2570 // the scan covers only the live suffix — a subset — so exact equality
2571 // is impossible. The subset relation still bounds the metadata from
2572 // BELOW: totals must be at least the suffix totals and the key range
2573 // must contain the scanned suffix. Blessing understated totals would
2574 // let blob GC's dead-file arithmetic reclaim a file whose uncounted
2575 // frames are still referenced.
2576 let range_contains = match (&first_key, &prev) {
2577 (Some(first), Some((last, _))) => {
2578 comparator.compare(meta.key_range.min().as_ref(), first.as_ref())
2579 != core::cmp::Ordering::Greater
2580 && comparator.compare(last.as_ref(), meta.key_range.max().as_ref())
2581 != core::cmp::Ordering::Greater
2582 }
2583 // An empty suffix constrains nothing.
2584 _ => true,
2585 };
2586 if meta.item_count < count
2587 || meta.total_uncompressed_bytes < uncompressed_total
2588 || meta.total_compressed_bytes < compressed_total
2589 || !range_contains
2590 {
2591 log::warn!(
2592 "blob file {blob_id} at {}: metadata understates the scanned live \
2593 suffix (meta: {} items / {} uncompressed bytes; suffix: {count} / \
2594 {uncompressed_total})",
2595 path.display(),
2596 meta.item_count,
2597 meta.total_uncompressed_bytes,
2598 );
2599 return Ok(None);
2600 }
2601 }
2602 Ok(Some(BlobLiveTotals {
2603 items: count,
2604 uncompressed_bytes: uncompressed_total,
2605 compressed_bytes: compressed_total,
2606 }))
2607}
2608
2609/// Whether any of `table`'s blob indirections points BELOW a recovered punched
2610/// blob file's live-data frontier — i.e. into its zeroed prefix. The
2611/// id-presence dependency check cannot see this case: the blob EXISTS, but a
2612/// pre-relocation SST file left behind by a crash still holds handles into the
2613/// prefix the relocation punched, and publishing it would resolve those reads
2614/// into zeroed bytes. Returns the first offending handle's description, or
2615/// `None` when every handle lands in live blob data. Called only when at least
2616/// one recovered blob file carries a frontier, so the sequential entry scan
2617/// costs nothing on the common path.
2618#[cfg(feature = "std")]
2619fn handle_below_blob_frontier(
2620 table: &Table,
2621 frontiers: &crate::HashMap<crate::vlog::BlobFileId, u64>,
2622) -> crate::Result<Option<String>> {
2623 use crate::coding::Decode;
2624
2625 for entry in table.scan()? {
2626 let entry = entry?;
2627 let handles = if entry.key.value_type == crate::ValueType::Indirection {
2628 let mut cursor = &entry.value[..];
2629 vec![crate::blob_tree::handle::BlobIndirection::decode_from(
2630 &mut cursor,
2631 )?]
2632 } else if entry.key.value_type == crate::ValueType::CellRow {
2633 crate::blob_tree::field_row::row_refs(&entry.value)?
2634 .into_iter()
2635 .map(|(ind, _)| ind)
2636 .collect()
2637 } else {
2638 continue;
2639 };
2640 for ind in handles {
2641 if let Some(&frontier) = frontiers.get(&ind.vhandle.blob_file_id)
2642 && ind.vhandle.offset < frontier
2643 {
2644 return Ok(Some(format!(
2645 "blob handle into file {} at offset {} lies below its recovered \
2646 live-data frontier {frontier}",
2647 ind.vhandle.blob_file_id, ind.vhandle.offset,
2648 )));
2649 }
2650 }
2651 }
2652 Ok(None)
2653}
2654
2655/// One table the blob-handle stage keeps: the table, its fidelity, its
2656/// scanned source and that source's filesystem, and whether its handles need
2657/// rewriting.
2658#[cfg(feature = "std")]
2659type AdmittedTable = ((Table, Fidelity, PathBuf, Arc<dyn crate::fs::Fs>), bool);
2660
2661/// For each table of `tables`, the cell rows to make the owners of the
2662/// objects whose owning row the rebuild does not keep, `None` for a table
2663/// with none.
2664///
2665/// An object is owned by one row of its key, and only that row's drop charges
2666/// the object as garbage: an object whose owner went with a table set aside,
2667/// a block dropped or a row the handle rewrite loses, while a row borrowing it
2668/// stays, would never be counted again. The oldest row left that references
2669/// it becomes its owner, as a compaction hands ownership to the oldest row it
2670/// keeps. The rows a table's handle rewrite will lose are left out here, as
2671/// the rewrite leaves them out. A version a table and a redundant copy of it
2672/// both hold is named in both, so the copy the lineage pass keeps owns it.
2673#[cfg(feature = "std")]
2674fn owner_promotions(
2675 tables: &[AdmittedTable],
2676 rewrites: &crate::HashMap<crate::vlog::BlobFileId, crate::salvage::BlobFileRewrite>,
2677) -> crate::Result<Vec<Option<crate::salvage::OwnerPromotions>>> {
2678 /// A row referencing an object by the field in `column`.
2679 struct Holder {
2680 table: usize,
2681 key: UserKey,
2682 seqno: SeqNo,
2683 column: u16,
2684 }
2685 type Object = (crate::vlog::BlobFileId, u64);
2686
2687 // Only borrowed references can lose their owner, and they are rare: a
2688 // first pass finds them and the oldest of each object's borrowers.
2689 let mut borrowed: crate::HashMap<Object, (SeqNo, Vec<Holder>)> = crate::HashMap::default();
2690 for (index, ((table, ..), rewrite)) in tables.iter().enumerate() {
2691 if table
2692 .list_blob_file_references()?
2693 .is_none_or(|links| links.is_empty())
2694 {
2695 continue;
2696 }
2697 let global = table.global_seqno();
2698 visit_kept_refs(table, rewrite.then_some(rewrites), |key, seqno, field| {
2699 let Some((column, true)) = field.owner else {
2700 return Ok(());
2701 };
2702 let eff = seqno
2703 .checked_add(global)
2704 .ok_or(crate::Error::Unrecoverable)?;
2705 let holder = Holder {
2706 table: index,
2707 key: key.clone(),
2708 seqno,
2709 column,
2710 };
2711 let object = (field.file, field.offset);
2712 match borrowed.get_mut(&object) {
2713 Some((oldest, holders)) if eff == *oldest => holders.push(holder),
2714 Some((oldest, holders)) if eff < *oldest => {
2715 *oldest = eff;
2716 *holders = alloc::vec![holder];
2717 }
2718 Some(_) => {}
2719 None => {
2720 borrowed.insert(object, (eff, alloc::vec![holder]));
2721 }
2722 }
2723 Ok(())
2724 })?;
2725 }
2726 let mut promotions: Vec<Option<crate::salvage::OwnerPromotions>> =
2727 tables.iter().map(|_| None).collect();
2728 if borrowed.is_empty() {
2729 return Ok(promotions);
2730 }
2731
2732 // Then every owning reference into the files those objects are in.
2733 let files: crate::HashSet<crate::vlog::BlobFileId> =
2734 borrowed.keys().map(|&(file, _)| file).collect();
2735 for ((table, ..), rewrite) in tables {
2736 let linked = table
2737 .list_blob_file_references()?
2738 .is_some_and(|links| links.iter().any(|l| files.contains(&l.blob_file_id)));
2739 if !linked {
2740 continue;
2741 }
2742 visit_kept_refs(table, rewrite.then_some(rewrites), |_, _, field| {
2743 if field.owner.is_none_or(|(_, borrows)| !borrows) {
2744 borrowed.remove(&(field.file, field.offset));
2745 }
2746 Ok(())
2747 })?;
2748 }
2749
2750 for (_, (_, holders)) in borrowed {
2751 for holder in holders {
2752 if let Some(slot) = promotions.get_mut(holder.table) {
2753 slot.get_or_insert_with(Default::default).insert(
2754 holder.key,
2755 holder.seqno,
2756 holder.column,
2757 );
2758 }
2759 }
2760 }
2761 Ok(promotions)
2762}
2763
2764/// A blob reference a row holds: the object's file and offset, and for a
2765/// cell row's field its column and whether the row owns the object, `None`
2766/// for an indirection, which always does.
2767#[cfg(feature = "std")]
2768#[derive(Clone, Copy)]
2769struct HeldRef {
2770 file: crate::vlog::BlobFileId,
2771 offset: u64,
2772 /// The field's column and whether the row only BORROWS the object.
2773 owner: Option<(u16, bool)>,
2774}
2775
2776/// Calls `visit` with every blob reference of `table`'s rows, by the row's
2777/// key and local seqno, leaving out the rows the handle rewrite `rewrites`
2778/// loses: a row with a reference to a record that no longer exists, and every
2779/// older version of its key in the table, as the rewrite does.
2780#[cfg(feature = "std")]
2781fn visit_kept_refs(
2782 table: &Table,
2783 rewrites: Option<&crate::HashMap<crate::vlog::BlobFileId, crate::salvage::BlobFileRewrite>>,
2784 mut visit: impl FnMut(&UserKey, SeqNo, HeldRef) -> crate::Result<()>,
2785) -> crate::Result<()> {
2786 use crate::blob_tree::field_row::{RowCell, decode_row};
2787 use crate::coding::Decode;
2788 use crate::salvage::BlobFileRewrite;
2789
2790 let lost =
2791 |file: crate::vlog::BlobFileId, offset: u64| match rewrites.and_then(|r| r.get(&file)) {
2792 Some(BlobFileRewrite::Remap { offsets, .. }) => !offsets.contains_key(&offset),
2793 Some(BlobFileRewrite::DropBelow(frontier)) => offset < *frontier,
2794 None => false,
2795 };
2796 let mut headless: Option<UserKey> = None;
2797 let mut refs: Vec<HeldRef> = Vec::new();
2798 for entry in table.scan()? {
2799 let entry = entry?;
2800 let key = &entry.key.user_key;
2801 if headless
2802 .as_ref()
2803 .is_some_and(|h| crate::comparator::same_user_key(key, h))
2804 {
2805 continue;
2806 }
2807 headless = None;
2808 refs.clear();
2809 match entry.key.value_type {
2810 crate::ValueType::Indirection => {
2811 let mut cursor = &entry.value[..];
2812 let ind = crate::blob_tree::handle::BlobIndirection::decode_from(&mut cursor)?;
2813 refs.push(HeldRef {
2814 file: ind.vhandle.blob_file_id,
2815 offset: ind.vhandle.offset,
2816 owner: None,
2817 });
2818 }
2819 crate::ValueType::CellRow => {
2820 for field in decode_row(&entry.value)? {
2821 if let RowCell::Ref { indirection, owner } = field.cell {
2822 refs.push(HeldRef {
2823 file: indirection.vhandle.blob_file_id,
2824 offset: indirection.vhandle.offset,
2825 owner: Some((field.column, !owner)),
2826 });
2827 }
2828 }
2829 }
2830 _ => continue,
2831 }
2832 if refs.iter().any(|r| lost(r.file, r.offset)) {
2833 headless = Some(key.clone());
2834 continue;
2835 }
2836 for &held in &refs {
2837 visit(key, entry.key.seqno, held)?;
2838 }
2839 }
2840 Ok(())
2841}
2842
2843/// What the blob directory scan recovered, and how, for the rebuilt manifest.
2844#[cfg(feature = "std")]
2845struct BlobRecovery {
2846 /// Blob files to record in the manifest (whole, punched, or salvaged).
2847 files: Vec<crate::vlog::BlobFile>,
2848 /// Files LEFT OUT of the manifest, with the reason each was set aside.
2849 unreadable: UnreadableFiles,
2850 /// Handle rewrites for reshaped files (salvaged / punched frontiers).
2851 rewrites: crate::HashMap<crate::vlog::BlobFileId, crate::salvage::BlobFileRewrite>,
2852 /// Garbage accounting seeded for punched files: the consumed prefix is
2853 /// stale by construction but can never be observed by a future
2854 /// compaction, so without this seed `is_dead` could never retire the
2855 /// file.
2856 frag: crate::blob_tree::FragmentationMap,
2857 /// Damaged originals whose salvaged replacement (a FRESH id) is in the
2858 /// rebuilt manifest, with a note describing what was recovered. They are
2859 /// still what the not-yet-rewritten SSTs reference, so the caller sets
2860 /// them aside only AFTER `persist_version`: doing it earlier would let a
2861 /// failed repair leave tables pointing at a blob id that no longer
2862 /// exists, and the retry would set those tables aside as unrecoverable.
2863 stale: Vec<(PathBuf, String)>,
2864 /// Files the rebuilt manifest will not name: a foreign name, a duplicate id,
2865 /// a file whose metadata cannot be read. Removed AFTER the commit, for the
2866 /// same reason `stale` is — the scan must leave the directory exactly as a
2867 /// retry expects to find it.
2868 discard: Vec<(PathBuf, String)>,
2869 /// HEALTHY exclusions (a valid physically distinct duplicate of a kept
2870 /// id): reported through the report's `excluded_files`, never the
2871 /// unreadable counts.
2872 excluded: Vec<(PathBuf, String)>,
2873}
2874
2875/// Discovers the blob files of a KV-separated tree for `repair` by scanning the
2876/// single `blobs/` folder, with no manifest id list to filter against.
2877///
2878/// Mirrors the table scan in [`repair_tree`]: a non-numeric name is recorded for
2879/// removal after the commit (the reopened tree's blob recovery parses every name
2880/// and would abort on a bad one), as is a blob file that cannot be checksummed
2881/// or whose metadata is unreadable. The recovered checksum is the whole-file
2882/// XXH3-128 digest, identical to the one the blob writer accumulated via
2883/// `ChecksummedWriter`, since blob files are written strictly sequentially.
2884fn recover_blob_files(
2885 config: &Config,
2886 published: &mut PublishedBlobReplacements<'_>,
2887 // Blob ids the recovered tables reference (a conservative SUPERSET: taken
2888 // before the blob-dependency filtering, so a table dropped later only
2889 // leaves an extra id here). An INVALID blob outside this set is left for
2890 // the post-commit sweep instead of being salvaged: the replacement would
2891 // be filtered out and deleted anyway, and under tight disk space the
2892 // pointless salvage can abort the whole repair with `StorageFull`.
2893 referenced: &crate::HashSet<crate::vlog::BlobFileId>,
2894 // The CLEANLY-loading manifest's committed blob frontiers, when one
2895 // exists. Each is the AUTHORITATIVE `[data_start, frontier)` boundary of a
2896 // tight-space-punched blob file — the same authority its `restrictions`
2897 // twin carries for SSTs — so it is preferred over re-deriving the frontier
2898 // from the on-disk punch geometry, which a backend that cannot report
2899 // extent allocation gets wrong (reporting a punched file whole, whose
2900 // handles then dereference the hole).
2901 committed_frontiers: Option<&crate::HashMap<crate::vlog::BlobFileId, u64>>,
2902 committed_checksums: Option<&crate::HashMap<crate::vlog::BlobFileId, crate::Checksum>>,
2903) -> crate::Result<BlobRecovery> {
2904 let blobs_folder = config.path.join(crate::file::BLOBS_FOLDER);
2905 let mut blob_files: Vec<crate::vlog::BlobFile> = Vec::new();
2906 let mut unreadable: UnreadableFiles = Vec::new();
2907 let mut discard: Vec<(PathBuf, String)> = Vec::new();
2908 let mut excluded: Vec<(PathBuf, String)> = Vec::new();
2909 // How referencing SSTs' handles must be rewritten for the blob files this
2910 // scan RESHAPED: `Remap` for a file salvaged into a compacted copy,
2911 // `DropBelow` for an intact file recovered with a punched frontier. Empty
2912 // on the common path.
2913 let mut rewrites: crate::HashMap<crate::vlog::BlobFileId, crate::salvage::BlobFileRewrite> =
2914 crate::HashMap::default();
2915 // Damaged originals whose salvaged replacement is in the rebuilt manifest.
2916 // They are still referenced by the not-yet-rewritten SSTs, so they are set
2917 // aside only AFTER the manifest commit — see `BlobRecovery::stale`.
2918 let mut stale_originals: Vec<(PathBuf, String)> = Vec::new();
2919 let mut frag = crate::blob_tree::FragmentationMap::default();
2920
2921 // No `blobs/` folder = no blob files (a blob tree that never spilled a value
2922 // to the value log). Nothing to recover; the manifest records an empty list.
2923 if !config.fs.exists(&blobs_folder)? {
2924 return Ok(BlobRecovery {
2925 files: blob_files,
2926 unreadable,
2927 rewrites,
2928 frag,
2929 stale: stale_originals,
2930 discard,
2931 excluded,
2932 });
2933 }
2934
2935 // Collect and ORDER the candidates before recovering: `read_dir` order is
2936 // FS-dependent, and duplicate-id resolution below must be deterministic.
2937 // Per id, the writer's own `id.to_string()` spelling is the canonical file
2938 // and sorts first, so a foreign alternate spelling (`01` for id 1) can
2939 // never displace it regardless of directory iteration order.
2940 let mut candidates: Vec<(crate::vlog::BlobFileId, PathBuf, String)> = Vec::new();
2941 for dirent in config.fs.read_dir(&blobs_folder)? {
2942 let crate::fs::FsDirEntry {
2943 path: blob_path,
2944 file_name,
2945 is_dir,
2946 } = dirent;
2947
2948 if is_dir {
2949 continue;
2950 }
2951
2952 // The file counts toward the progress percentage as soon as its
2953 // processing starts — including the entries the early arms below
2954 // reject (a crashed salvage temp, a non-numeric name), or a finished
2955 // repair would sit below 100% (`bytes_processed < bytes_total`).
2956 // Mirrors the table scan; best-effort stat, like the total.
2957 if let Some(p) = &config.recovery_progress
2958 && let Ok(meta) = config.fs.metadata(&blob_path)
2959 {
2960 p.add_bytes_processed(meta.len);
2961 }
2962
2963 let blob_id = match crate::file::BlobDirEntry::classify(&file_name) {
2964 crate::file::BlobDirEntry::Blob(id) => id,
2965 // A crashed earlier repair's in-progress salvage copy: published by
2966 // an atomic rename, so a survivor is never referenced and never
2967 // authoritative. Removed here (and re-salvaged from the original
2968 // below if that file still fails validation).
2969 //
2970 // A removal failure fails the repair: the temp is outside the
2971 // rebuilt manifest, so the next open classifies it as an orphan
2972 // and its sweep hits the same removal failure — reporting success
2973 // for a tree that cannot open would be a lie. Quarantine is not
2974 // an out (it preserves damaged DATA; a temp is discardable
2975 // garbage, and a directory refusing removal refuses the rename
2976 // too). Retry after the filesystem is fixed.
2977 crate::file::BlobDirEntry::SalvageTmp(_) => {
2978 remove_temp(config, &blob_path)?;
2979 continue;
2980 }
2981 // Not a shape the engine names: not part of the inventory being
2982 // rebuilt, and not the repair's to remove.
2983 crate::file::BlobDirEntry::Foreign => {
2984 log::debug!(
2985 "repair: ignoring {} in the blobs folder: not an engine file",
2986 blob_path.display(),
2987 );
2988 continue;
2989 }
2990 };
2991 candidates.push((blob_id, blob_path, file_name));
2992 }
2993 // Fresh-id allocator for salvaged replacements. Taken over ALL candidates
2994 // before any is processed, so an id handed out here can never collide with
2995 // a file this scan has not reached yet — AND over every id the recovered
2996 // tables still REFERENCE: an SST may point at a blob file that no longer
2997 // exists on disk, and allocating that missing id to an unrelated
2998 // replacement would let the later dependency check find the id present
2999 // and keep the SST, whose handles then resolve against the wrong file's
3000 // records. Repair runs single-threaded on a tree nobody else has open,
3001 // so no other allocator competes.
3002 //
3003 // `None` once the space is exhausted, and the failure is raised only where
3004 // an id is actually needed — a healthy tree holding a `u64::MAX` blob is
3005 // still perfectly repairable. Wrapping instead would hand out an id an
3006 // existing file already holds, and publishing the replacement onto that
3007 // name would destroy an unrelated original BEFORE the manifest commit, so
3008 // the loss would not even surface as a failed repair.
3009 let mut next_blob_id: Option<crate::vlog::BlobFileId> = candidates
3010 .iter()
3011 .map(|(id, _, _)| *id)
3012 .chain(referenced.iter().copied())
3013 .max()
3014 .map_or(Some(0), |max| max.checked_add(1));
3015 candidates.sort_by(|a, b| {
3016 a.0.cmp(&b.0)
3017 .then_with(|| {
3018 (a.1 != blobs_folder.join(a.0.to_string()))
3019 .cmp(&(b.1 != blobs_folder.join(b.0.to_string())))
3020 })
3021 .then_with(|| a.2.cmp(&b.2))
3022 });
3023
3024 // The path recovered for each id, for the duplicate-vs-alias decision.
3025 let mut kept_paths: crate::HashMap<crate::vlog::BlobFileId, PathBuf> =
3026 crate::HashMap::default();
3027
3028 // Whether a file opens on its OWN bytes, for the duplicate verdict below.
3029 // A duplicate contributes nothing to the rebuild, but the report's two
3030 // channels make different claims about it, and health inferred from the
3031 // RETAINED copy would file a rotting duplicate as healthy. A FULLY PUNCHED
3032 // file is not damaged — its relocation completed — so it counts as clean.
3033 // The live-data frontier of one file: the manifest's COMMITTED value when
3034 // it survived — exact, free, and independent of whether the backend can
3035 // attribute zeros to a hole — else derived from the punch geometry. Single
3036 // source for the recovery walk and the duplicate verdict alike.
3037 let resolve_frontier = |blob_id, blob_path: &std::path::Path| -> crate::Result<BlobFrontier> {
3038 match committed_frontiers.and_then(|m| m.get(&blob_id).copied()) {
3039 Some(committed) => Ok(BlobFrontier::Punched(committed)),
3040 None => derive_blob_frontier(&config.fs, blob_path, blob_id),
3041 }
3042 };
3043
3044 let opens_cleanly = |blob_id, blob_path: &std::path::Path| -> crate::Result<()> {
3045 let frontier = match resolve_frontier(blob_id, blob_path)? {
3046 BlobFrontier::Whole => 0,
3047 BlobFrontier::Punched(f) => f,
3048 BlobFrontier::FullyConsumed => return Ok(()),
3049 };
3050 // The digest is recomputed from THESE bytes, so it can only prove the
3051 // file is self-consistent — never that it is undamaged. Recovery then
3052 // parses the metadata section alone, so a rotted value frame sails
3053 // through both. Walk the frames, exactly as a retained file's does.
3054 let checksum = crate::Checksum::from_raw(compute_table_checksum_from(
3055 &*config.fs,
3056 blob_path,
3057 frontier,
3058 )?);
3059 let handle = crate::vlog::recover_blob_file_from(
3060 blob_path,
3061 blob_id,
3062 checksum,
3063 0,
3064 &config.fs,
3065 frontier,
3066 #[cfg(zstd_any)]
3067 &config.current_zstd_dictionaries(),
3068 )?;
3069 // Without the dictionary no copy can be verified. For an id a recovered
3070 // table references that stops the repair, as it does in the recovery
3071 // below, since the choice decides which copy survives. For one nothing
3072 // references no copy survives either way: it simply does not verify,
3073 // and every copy takes the unreferenced path to the sweep.
3074 #[cfg(zstd_any)]
3075 let dict = match blob_file_dictionary(config, handle.compression()) {
3076 Ok(dict) => dict,
3077 Err(e) if referenced.contains(&blob_id) => return Err(e),
3078 Err(_) => {
3079 return Err(crate::Error::InvalidHeader(
3080 "blob file names a dictionary the tree does not hold",
3081 ));
3082 }
3083 };
3084 match validate_blob_frames(
3085 config,
3086 blob_path,
3087 blob_id,
3088 frontier,
3089 &handle,
3090 #[cfg(zstd_any)]
3091 dict.as_ref(),
3092 )? {
3093 Some(_) => Ok(()),
3094 None => Err(crate::Error::InvalidHeader(
3095 "blob frame validation failed on this copy",
3096 )),
3097 }
3098 };
3099
3100 // Sightings of one id are contiguous (the sort keys on id first) and the
3101 // CANONICAL spelling leads each group. That order is wrong when the leader
3102 // is damaged: the loop below would salvage it into a fresh id, record that
3103 // lossy replacement as the incumbent, and then discard a later INTACT copy
3104 // as a redundant duplicate — committing a partial rewrite of data that was
3105 // available in full. Promote a cleanly-opening sighting to the front of its
3106 // group so salvage runs only when no copy of the id is whole.
3107 let candidates = {
3108 let mut ordered: Vec<(crate::vlog::BlobFileId, PathBuf, String)> =
3109 Vec::with_capacity(candidates.len());
3110 let mut rest = candidates.into_iter().peekable();
3111 while let Some(first) = rest.next() {
3112 let blob_id = first.0;
3113 let mut group = vec![first];
3114 while rest.peek().is_some_and(|c| c.0 == blob_id) {
3115 if let Some(c) = rest.next() {
3116 group.push(c);
3117 }
3118 }
3119 if group.len() > 1 {
3120 // Opening cleanly proves a copy is SELF-consistent, never that
3121 // it is the one the manifest named: two copies of an id can
3122 // both be whole and hold different generations, and adopting
3123 // the wrong one re-stamps a fresh checksum over stale bytes
3124 // that existing SST handles then resolve against. When the
3125 // committed digest survived, it decides.
3126 let committed = committed_checksums.and_then(|m| m.get(&blob_id).copied());
3127 let mut authoritative = None;
3128 let mut whole = None;
3129 for (k, candidate) in group.iter().enumerate() {
3130 if let Some(expected) = committed
3131 && authoritative.is_none()
3132 {
3133 let frontier = match resolve_frontier(blob_id, &candidate.1)? {
3134 BlobFrontier::Whole => Some(0),
3135 BlobFrontier::Punched(f) => Some(f),
3136 // Nothing live is left to digest.
3137 BlobFrontier::FullyConsumed => None,
3138 };
3139 if let Some(frontier) = frontier {
3140 match compute_table_checksum_from(&*config.fs, &candidate.1, frontier) {
3141 Ok(d) if crate::Checksum::from_raw(d) == expected => {
3142 authoritative = Some(k);
3143 continue;
3144 }
3145 // A fault in the ENVIRONMENT decides nothing
3146 // and the decision here costs the loser its
3147 // file; a digest that simply differs, or a
3148 // read that fails on the bytes, both mean
3149 // "not the manifest's copy".
3150 Err(e) if is_environmental(&e) => return Err(e),
3151 Ok(_) | Err(_) => {}
3152 }
3153 }
3154 }
3155 if whole.is_none() {
3156 match opens_cleanly(blob_id, &candidate.1) {
3157 Ok(()) => whole = Some(k),
3158 // Not evidence about these bytes, and the choice it
3159 // would steer decides which copy survives.
3160 Err(e) if is_environmental(&e) => return Err(e),
3161 Err(_) => {}
3162 }
3163 }
3164 }
3165 // The manifest's own copy first; without one (no clean
3166 // manifest, or none of the copies reproduces its digest) an
3167 // intact copy still beats a salvage of a damaged leader.
3168 if let Some(k) = authoritative.or(whole) {
3169 group.swap(0, k);
3170 }
3171 }
3172 ordered.extend(group);
3173 }
3174 ordered
3175 };
3176
3177 for (blob_id, blob_path, _file_name) in candidates {
3178 if let Some(kept) = kept_paths.get(&blob_id) {
3179 // A second directory entry for an already-recovered id. An ALIAS
3180 // (symlink / case-folded spelling of the SAME physical file) is
3181 // skipped silently. A DISTINCT physical file must go: the manifest
3182 // records one checksum per id, and a stale duplicate left in
3183 // `blobs/` would race the kept file for reads on the next open
3184 // (directory iteration order picks the physical file).
3185 if same_physical_file(&*config.fs, kept, &*config.fs, &blob_path)? {
3186 continue;
3187 }
3188 // Verify THIS copy before choosing the report channel: an
3189 // exclusion claims the file opened and verified, and calling an
3190 // unchecked duplicate healthy hides the corruption signal.
3191 match opens_cleanly(blob_id, &blob_path) {
3192 Ok(()) => {
3193 let reason = format!("duplicate of blob file id {blob_id}");
3194 discard.push((blob_path.clone(), reason.clone()));
3195 excluded.push((blob_path, reason));
3196 }
3197 // A fault in the ENVIRONMENT is not evidence about these bytes.
3198 Err(e) if is_environmental(&e) => return Err(e),
3199 Err(e) => {
3200 let reason = format!(
3201 "damaged duplicate of blob file id {blob_id} (kept copy is intact): {e}"
3202 );
3203 discard.push((blob_path.clone(), reason.clone()));
3204 unreadable.push((blob_path, reason));
3205 }
3206 }
3207 continue;
3208 }
3209
3210 // Pre-commit file boundary: safe to abort. The caller's
3211 // `PublishedBlobReplacements` guard unwinds the replacements this run
3212 // already published under fresh ids — unlike the read-only table
3213 // scan, a successful blob salvage renames its copy to a normal
3214 // numeric name before the commit, and leaving those behind would make
3215 // the retry re-create each one beside its orphan (under tight disk
3216 // space exactly the sequence that fails).
3217 check_cancel(config)?;
3218 if let Some(p) = &config.recovery_progress {
3219 p.blob_file_discovered();
3220 }
3221
3222 // A tight-space-punched blob records its live-data frontier only in
3223 // the manifest; with the manifest lost, re-derive it from the punch
3224 // geometry so the rebuilt manifest restores the restriction (the
3225 // snapshot encoder persists it from the recovered `live_data_start`).
3226 // Rebuilding with frontier 0 would instead leave a later relocation
3227 // scan starting inside the punched (zeroed) prefix. An unpunched file
3228 // short-circuits to 0 on its first (non-zero) data byte.
3229 // Persistent per-file failure: the rebuilt manifest omits this file, so
3230 // it is recorded for removal after the commit and reported unreadable.
3231 // The file must not be both omitted and left in place — that is a tree
3232 // whose next open has an orphan to sweep and may fail doing it.
3233 let discard_unreadable =
3234 |blob_path: PathBuf,
3235 e: &crate::Error,
3236 unreadable: &mut UnreadableFiles,
3237 discard: &mut Vec<(PathBuf, String)>| {
3238 discard.push((blob_path.clone(), e.to_string()));
3239 unreadable.push((blob_path, e.to_string()));
3240 };
3241
3242 // The committed frontier, when the manifest loaded cleanly, is exact
3243 // and free: no geometry walk, and no dependence on the backend being
3244 // able to attribute zeros to a hole (which, unanswered, reports a
3245 // punched file WHOLE — publishing handles that dereference the hole).
3246 // Everything downstream — identity, frame validation, the handle
3247 // rewrite — runs on it exactly as on a derived one.
3248 let frontier = match resolve_frontier(blob_id, &blob_path) {
3249 Ok(BlobFrontier::Whole) => 0,
3250 Ok(BlobFrontier::Punched(f)) => f,
3251 Ok(BlobFrontier::FullyConsumed) => {
3252 // Every frame is punched away: the relocation that consumed
3253 // this file completed, only its removal lagged the crash.
3254 // QUEUE that lagged drop for after the commit instead of
3255 // finishing it here: the pre-commit scan is read-only on
3256 // purpose — an abort before the commit (a cancellation, a
3257 // transient failure on a later file) must leave the directory
3258 // exactly as found, and the OLD manifest of an explicitly
3259 // invoked repair over an openable tree may still name this
3260 // file. Publishing an empty-suffix handle instead is not an
3261 // option either: whole-file metadata over zero live frames is
3262 // a file blob GC's stale-byte arithmetic can never retire
3263 // (its frames are already gone, so the stale count never
3264 // reaches the recorded totals). No live data is discarded:
3265 // the walk proved the whole data section reads as zeros. The
3266 // post-commit removal failing fails the repair, exactly as
3267 // the immediate removal used to.
3268 log::info!(
3269 "blob file {blob_id} at {}: its punch consumed every frame — \
3270 queueing the relocation's lagged file drop for after the commit",
3271 blob_path.display(),
3272 );
3273 discard.push((
3274 blob_path,
3275 "fully punched blob file: a completed relocation's lagged drop".to_string(),
3276 ));
3277 continue;
3278 }
3279 // A read failure in the ENVIRONMENT is retryable: recording the blob
3280 // unreadable commits a manifest without the still-in-place file,
3281 // which the next open's orphan sweep then DELETES — permanent value
3282 // loss from a fixable failure. Propagate so a retry re-reads it,
3283 // mirroring the table-recovery path.
3284 Err(e) if is_environmental(&e) => return Err(e),
3285 Err(e) => {
3286 discard_unreadable(blob_path, &e, &mut unreadable, &mut discard);
3287 continue;
3288 }
3289 };
3290
3291 // IDENTITY before content: the id under which this file will be
3292 // published comes from its FILENAME, but the metadata records the id
3293 // it was written as. A mismatch means a renamed or swapped file — not
3294 // damaged content — and publishing it under the filename's id would
3295 // resolve existing SST handles into foreign frames (a read fails on
3296 // the key cross-check, or, when the foreign frame happens to hold the
3297 // same key, silently serves another generation's value). Salvage is
3298 // the WRONG remedy here (it would re-emit the foreign records under
3299 // the filename's id, laundering the swap); set the file aside.
3300 // Placeholder-checksum open (stored, never verified). Same
3301 // transient/persistent split as every other per-file step: an
3302 // unreadable meta section sets THIS file aside, never the repair.
3303 // Read ONCE and kept: the identity check needs the stored id and the
3304 // frame validation below needs the compression descriptor, and both
3305 // live in this same meta section.
3306 let handle = match crate::vlog::recover_blob_file(
3307 &blob_path,
3308 blob_id,
3309 crate::Checksum::from_raw(0),
3310 0,
3311 &config.fs,
3312 #[cfg(zstd_any)]
3313 &config.current_zstd_dictionaries(),
3314 ) {
3315 Ok(handle) => handle,
3316 Err(e) if is_environmental(&e) => return Err(e),
3317 Err(e) => {
3318 discard_unreadable(blob_path, &e, &mut unreadable, &mut discard);
3319 continue;
3320 }
3321 };
3322 let stored = handle.meta().id;
3323 if stored != blob_id {
3324 let e = crate::Error::InvalidHeader(
3325 "blob file's stored metadata id disagrees with its file name",
3326 );
3327 log::warn!(
3328 "blob file at {}: metadata records id {stored}, file name says \
3329 {blob_id} — a renamed or swapped file; setting it aside",
3330 blob_path.display(),
3331 );
3332 discard_unreadable(blob_path, &e, &mut unreadable, &mut discard);
3333 continue;
3334 }
3335
3336 // Validate the live frame range BEFORE recording a digest: hashing
3337 // damaged frames would restamp (launder) the corruption past every
3338 // later integrity check while reads of the affected values still
3339 // fail. An invalid file is SALVAGED instead of blessed or thrown away
3340 // whole: its surviving records are re-emitted into a compacted
3341 // replacement and the offset relocation is recorded so the referencing
3342 // SSTs are rewritten onto the new offsets.
3343 //
3344 // The replacement is written under a FRESH blob file id, so it never
3345 // displaces the damaged original: the original keeps its id and its
3346 // bytes, and only the manifest commit makes the replacement live. A
3347 // crash at any point before that commit therefore leaves the tree
3348 // exactly as it was found, and the retry re-derives everything from
3349 // the untouched original — no relocation record has to survive the
3350 // crash, because nothing was relocated in place. The original is
3351 // removed only AFTER the commit: removing it earlier would make a
3352 // crashed attempt leave SSTs referencing a blob id that no longer
3353 // exists, and the retry would record those tables unrecoverable.
3354 //
3355 // The dictionary the file's descriptor names comes first. A missing one
3356 // is not damage: the frames would all fail to decode and the file would
3357 // be salvaged into nothing. One a recovered table references stops the
3358 // repair naming the id, before anything is committed, so the operator
3359 // supplies the dictionary and re-runs. One nothing references holds no
3360 // value anyone can reach, and goes to the sweep like any other.
3361 #[cfg(zstd_any)]
3362 let dict = match blob_file_dictionary(config, handle.compression()) {
3363 Ok(dict) => dict,
3364 Err(e) if referenced.contains(&blob_id) => return Err(e),
3365 Err(e) => {
3366 log::debug!(
3367 "blob file {blob_id} names a dictionary the tree does not hold \
3368 ({e}) and no recovered table references it; leaving it for the \
3369 post-commit sweep"
3370 );
3371 discard.push((
3372 blob_path,
3373 "names a dictionary the tree does not hold, and no recovered table \
3374 references it"
3375 .to_string(),
3376 ));
3377 continue;
3378 }
3379 };
3380 let Some(live) = validate_blob_frames(
3381 config,
3382 &blob_path,
3383 blob_id,
3384 frontier,
3385 &handle,
3386 #[cfg(zstd_any)]
3387 dict.as_ref(),
3388 )?
3389 else {
3390 // An INVALID blob no recovered table references never reaches the
3391 // manifest: salvaging it would allocate a fresh id and burn disk
3392 // space on a replacement the reference filter deletes anyway —
3393 // and under tight space that pointless salvage can abort the
3394 // whole repair with `StorageFull`. Leave it for the post-commit
3395 // sweep instead; nothing reachable is lost.
3396 if !referenced.contains(&blob_id) {
3397 log::debug!(
3398 "blob file {blob_id} is invalid and referenced by no recovered \
3399 table; skipping its salvage and leaving it for the post-commit \
3400 sweep"
3401 );
3402 discard.push((
3403 blob_path,
3404 "invalid and referenced by no recovered table; salvage skipped".to_string(),
3405 ));
3406 continue;
3407 }
3408 let Some(new_id) = next_blob_id else {
3409 log::error!(
3410 "blob file {blob_id} needs salvaging but the blob id space is \
3411 exhausted: there is no fresh name to publish a replacement \
3412 under, and reusing one would destroy an existing file"
3413 );
3414 return Err(crate::Error::Unrecoverable);
3415 };
3416 next_blob_id = new_id.checked_add(1);
3417 let temp = blobs_folder.join(format!("{new_id}.salvage-tmp"));
3418 remove_temp(config, &temp)?;
3419 let salvage = (|| -> crate::Result<Option<(crate::vlog::BlobFile, crate::salvage::BlobSalvageReport)>> {
3420 let report = crate::salvage::salvage_blob_file_on(
3421 &blob_path,
3422 temp.clone(),
3423 &config.fs,
3424 // The OUTPUT's id: the replacement is stamped as the fresh
3425 // file it is, so its metadata id matches the name it will
3426 // be published under.
3427 new_id,
3428 &config.comparator,
3429 frontier,
3430 // Repair has the tree's dictionary context, so a
3431 // dictionary-compressed blob salvages its intact frames
3432 // instead of being set aside whole. The id comes from the
3433 // SOURCE's own descriptor, so a file written under an
3434 // earlier dictionary resolves to that one.
3435 #[cfg(zstd_any)]
3436 dict.as_ref(),
3437 // Stamped on the tree's clock, as the blob files it writes are.
3438 config.clock.as_ref(),
3439 )?;
3440 let Some(salvaged_path) = report.salvaged_path.clone() else {
3441 return Ok(None);
3442 };
3443 let checksum = crate::Checksum::from_raw(compute_table_checksum(
3444 &*config.fs,
3445 &salvaged_path,
3446 )?);
3447 let held = held_descriptors(config);
3448 let bf = crate::vlog::recover_blob_file_cached(
3449 &salvaged_path,
3450 new_id,
3451 checksum,
3452 held.as_ref().map_or(0, |(_, tree_id)| *tree_id),
3453 &config.fs,
3454 0,
3455 #[cfg(zstd_any)]
3456 &config.current_zstd_dictionaries(),
3457 held.as_ref().map(|(cache, _)| cache),
3458 )?;
3459 Ok(Some((bf, report)))
3460 })();
3461 let (bf, report) = match salvage {
3462 Ok(Some(pair)) => pair,
3463 // Nothing recoverable, or a persistent failure: report it and
3464 // queue its removal — exactly like any other unreadable blob.
3465 Ok(None) => {
3466 remove_temp(config, &temp)?;
3467 let e = crate::Error::InvalidHeader(
3468 "blob value frames failed validation and no record was recoverable",
3469 );
3470 discard_unreadable(blob_path, &e, &mut unreadable, &mut discard);
3471 continue;
3472 }
3473 Err(e) if is_environmental(&e) => {
3474 // Nothing was published; dropping the temp restores the
3475 // pre-repair state exactly, so the retry re-salvages and
3476 // re-derives the remap. Best-effort is enough on this
3477 // ERROR path: no manifest is committed, and the retry's
3478 // candidate sweep sets a stuck temp aside.
3479 let _ = config.fs.remove_file(&temp);
3480 return Err(e);
3481 }
3482 Err(e) => {
3483 remove_temp(config, &temp)?;
3484 discard_unreadable(blob_path, &e, &mut unreadable, &mut discard);
3485 continue;
3486 }
3487 };
3488
3489 // Take the fresh id's own (free) name: nothing is displaced, so
3490 // there is nothing to unwind and no window in which a retry could
3491 // mistake an unverified file for the live blob. Until the
3492 // manifest names it, the replacement is an unreferenced file that
3493 // the next open's orphan sweep would remove.
3494 let new_path = blobs_folder.join(new_id.to_string());
3495 if let Err(e) = config.fs.rename(&temp, &new_path) {
3496 // Best-effort on this ERROR path: no manifest is committed,
3497 // so a stuck temp is swept by the retry's candidate scan.
3498 let _ = config.fs.remove_file(&temp);
3499 return Err(e.into());
3500 }
3501 // Under the caller's guard from this moment: any pre-commit exit
3502 // removes the published file.
3503 published.publish(new_id);
3504 config
3505 .fs
3506 .sync_directory_with(&blobs_folder, config.sync_mode)?;
3507
3508 blob_files.push(bf);
3509 rewrites.insert(
3510 blob_id,
3511 crate::salvage::BlobFileRewrite::Remap {
3512 new_id,
3513 offsets: report.offset_remap.iter().copied().collect(),
3514 },
3515 );
3516 // The damaged original is left in place for now and set aside once
3517 // the manifest is committed: it is still what the not-yet-rewritten
3518 // SSTs reference, so removing it earlier would strand them if this
3519 // repair failed before committing.
3520 // Report only what the walk can actually account for. A
3521 // structural failure DESYNCHRONIZES the record stream: the walk
3522 // stops there and surrenders everything after it, so
3523 // `records_total` counts the prefix it managed to frame, not the
3524 // source's true population. Presenting that as "X of Y" would
3525 // claim a near-complete recovery while an unknown — possibly
3526 // enormous — tail went with it. Only when every loss was an
3527 // individually re-synced record is the total trustworthy.
3528 let surrendered_tail = report
3529 .dropped
3530 .iter()
3531 .any(|d| matches!(d.reason, crate::salvage::BlobDropReason::Corrupt(_)));
3532 let note = if surrendered_tail {
3533 format!(
3534 "{} records salvaged into blob file {new_id}; the record \
3535 stream then desynchronized and the remainder of the file \
3536 was surrendered, so the number of records lost with it is \
3537 not knowable",
3538 report.records_salvaged,
3539 )
3540 } else {
3541 format!(
3542 "{} of {} records salvaged into blob file {new_id} \
3543 (the rest failed their checksums)",
3544 report.records_salvaged, report.records_total,
3545 )
3546 };
3547 stale_originals.push((blob_path.clone(), note));
3548 kept_paths.insert(blob_id, blob_path);
3549 continue;
3550 };
3551
3552 // The digest covers the live region only — `[frontier, end)` for a
3553 // punched file, the whole file for `frontier == 0` — matching what
3554 // `reopen_restricted` records and what integrity checks recompute.
3555 let checksum = match compute_table_checksum_from(&*config.fs, &blob_path, frontier) {
3556 Ok(c) => crate::Checksum::from_raw(c),
3557 // Same transient/persistent split as the frontier probe above.
3558 Err(e) if is_environmental(&e) => return Err(e),
3559 Err(e) => {
3560 discard_unreadable(blob_path, &e, &mut unreadable, &mut discard);
3561 continue;
3562 }
3563 };
3564
3565 // Held until the rebuilt manifest is published, as a recovered table
3566 // is, so its descriptor goes through the repair's cache (see
3567 // `held_recover_params`).
3568 let held = held_descriptors(config);
3569 match crate::vlog::recover_blob_file_cached(
3570 &blob_path,
3571 blob_id,
3572 checksum,
3573 held.as_ref().map_or(0, |(_, tree_id)| *tree_id),
3574 &config.fs,
3575 frontier,
3576 #[cfg(zstd_any)]
3577 &config.current_zstd_dictionaries(),
3578 held.as_ref().map(|(cache, _)| cache),
3579 ) {
3580 Ok(bf) => {
3581 if frontier > 0 {
3582 // A punched-but-intact file: a stale handle below its
3583 // frontier (a pre-relocation SST left behind by a crash)
3584 // must be dropped by the table-rewrite stage.
3585 rewrites.insert(
3586 blob_id,
3587 crate::salvage::BlobFileRewrite::DropBelow(frontier),
3588 );
3589 // Seed the punched prefix's garbage: the recovered handle
3590 // keeps its WHOLE-FILE metadata totals, while the prefix's
3591 // frames can never be observed by a future compaction. An
3592 // empty fragmentation map would pin the recorded stale
3593 // bytes below the totals forever, so `is_dead` could
3594 // never retire the file even after every suffix handle is
3595 // gone. The differences cannot underflow: validation just
3596 // proved the metadata totals are at least the suffix's.
3597 let meta = bf.meta();
3598 let prefix_items = meta.item_count - live.items;
3599 let prefix_len =
3600 usize::try_from(prefix_items).map_err(|_| crate::Error::Unrecoverable)?;
3601 frag.insert(
3602 blob_id,
3603 crate::blob_tree::FragmentationEntry::new(
3604 prefix_len,
3605 meta.total_uncompressed_bytes - live.uncompressed_bytes,
3606 meta.total_compressed_bytes - live.compressed_bytes,
3607 ),
3608 );
3609 }
3610 kept_paths.insert(blob_id, blob_path);
3611 blob_files.push(bf);
3612 }
3613 // Same transient/persistent split as the checksum read above.
3614 Err(e) if is_environmental(&e) => return Err(e),
3615 Err(e) => {
3616 discard_unreadable(blob_path, &e, &mut unreadable, &mut discard);
3617 }
3618 }
3619 }
3620
3621 Ok(BlobRecovery {
3622 files: blob_files,
3623 unreadable,
3624 rewrites,
3625 frag,
3626 stale: stale_originals,
3627 discard,
3628 excluded,
3629 })
3630}
3631
3632impl Config {
3633 /// Rebuilds the `MANIFEST` for the tree at this config's path from the SST
3634 /// files present on disk, then returns a [`RepairReport`].
3635 ///
3636 /// Use this only when a tree fails to open because its manifest is missing
3637 /// or corrupt but the SST files are intact. After a successful repair the
3638 /// tree opens normally; all recovered data is at L0 and a background
3639 /// compaction restructures it into proper levels (expect elevated I/O for a
3640 /// period proportional to the data size).
3641 ///
3642 /// Version edits the lost manifest carried are gone with it (the report's
3643 /// standing warning says so): in particular, a committed compaction whose
3644 /// filter removed EVERY record left no output to carry lineage, so its
3645 /// still-lingering inputs are republished as one consistent
3646 /// pre-compaction history — the filter's removals transiently reappear
3647 /// until the next compaction re-applies the standing policy. No operand
3648 /// is doubled by that window.
3649 ///
3650 /// # The report
3651 ///
3652 /// The rebuilt tree keeps the returned report until
3653 /// [`AbstractTree::acknowledge_repair_report`](crate::AbstractTree::acknowledge_repair_report)
3654 /// takes its id; until then [`Config::open_or_repair`] hands it out
3655 /// again, so a process that ends before the report was acted on loses
3656 /// nothing.
3657 ///
3658 /// # Exclusive access
3659 ///
3660 /// Repair writes and installs a fresh snapshot and removes the stale
3661 /// `edits-*` logs in place, so it requires exclusive access to the tree
3662 /// directory. It acquires the same cross-process directory lock as
3663 /// [`Config::open`] for the duration of the call: if another live instance
3664 /// holds the directory (open or repairing), this fails fast with
3665 /// [`crate::Error::Locked`] instead of corrupting that instance's manifest
3666 /// state. The lock can be disabled via
3667 /// [`Config::with_directory_lock`](crate::Config::with_directory_lock) for
3668 /// embedders enforcing exclusivity at a higher layer.
3669 ///
3670 /// # Errors
3671 ///
3672 /// Returns [`crate::Error::FeatureUnsupported`] for KV-separated (blob)
3673 /// trees, and propagates any I/O error from scanning the directory or
3674 /// writing the new manifest. Individual unreadable SSTs do not fail the
3675 /// repair; they are reported in [`RepairReport::unreadable_files`].
3676 ///
3677 /// # Examples
3678 ///
3679 /// ```
3680 /// use lsm_tree::{AbstractTree, Config, SequenceNumberCounter};
3681 ///
3682 /// let folder = tempfile::tempdir()?;
3683 /// let config = Config::new(
3684 /// &folder,
3685 /// SequenceNumberCounter::default(),
3686 /// SequenceNumberCounter::default(),
3687 /// );
3688 /// {
3689 /// let tree = config.clone().open()?;
3690 /// tree.insert("a", "1", 0);
3691 /// tree.flush_active_memtable(0)?;
3692 /// }
3693 ///
3694 /// let report = config.repair()?;
3695 /// println!("recovered {} tables, {} unreadable", report.recovered, report.unreadable);
3696 ///
3697 /// // `repair` borrows, so the same config opens the rebuilt tree. Once the
3698 /// // report's replay obligation is met, the tree stops owing it.
3699 /// let tree = config.open()?;
3700 /// assert!(tree.get("a", 1)?.is_some());
3701 /// assert!(tree.acknowledge_repair_report(report.id)?);
3702 /// # Ok::<(), lsm_tree::Error>(())
3703 /// ```
3704 pub fn repair(&self) -> crate::Result<RepairReport> {
3705 repair_tree(self, false, false)
3706 }
3707
3708 /// Like [`repair`](Self::repair), but when an SST fails whole-file recovery
3709 /// (`salvage = true`) it is block-salvaged instead of being left out: a fresh
3710 /// SST holding its recoverable blocks is built beside it, referenced by the
3711 /// rebuilt manifest, and swapped onto its name once that manifest is durable.
3712 ///
3713 /// A salvaged table may be missing the key ranges of its corrupt blocks
3714 /// (reported per file via [`RepairReport::salvaged`]); use this only as a
3715 /// last resort when losing the whole SST is worse than losing some keys.
3716 /// SSTs whose metadata, index, or SFA trailer is itself unreadable still
3717 /// cannot be salvaged and are reported unreadable.
3718 ///
3719 /// # Errors
3720 ///
3721 /// Returns an error if the directory lock cannot be taken or the rebuilt
3722 /// manifest cannot be persisted; per-file recovery / salvage failures are
3723 /// reported in the [`RepairReport`], not returned.
3724 ///
3725 /// # Examples
3726 ///
3727 /// ```no_run
3728 /// use lsm_tree::{Config, SequenceNumberCounter};
3729 ///
3730 /// let config = Config::new(
3731 /// "/var/lib/mydb",
3732 /// SequenceNumberCounter::default(),
3733 /// SequenceNumberCounter::default(),
3734 /// );
3735 /// let report = config.repair_with_salvage(true)?;
3736 /// println!(
3737 /// "recovered {} table(s), {} of them by salvage",
3738 /// report.recovered, report.salvaged,
3739 /// );
3740 /// # Ok::<(), lsm_tree::Error>(())
3741 /// ```
3742 pub fn repair_with_salvage(&self, salvage: bool) -> crate::Result<RepairReport> {
3743 repair_tree(self, salvage, false)
3744 }
3745
3746 /// Like [`repair_with_salvage`](Self::repair_with_salvage), but with an
3747 /// explicit `allow_resurrection` policy. When `false` (the default the other
3748 /// entry points use), recovery drops data whose visibility became ambiguous
3749 /// after a lost restriction bound or a lost / forged delete mask; when
3750 /// `true`, it keeps that data, accepting that superseded or deleted rows may
3751 /// reappear. Either setting yields a valid, openable tree; the flag is the
3752 /// ONLY recovery decision an operator makes.
3753 ///
3754 /// The flag governs AMBIGUOUS VISIBILITY, not LOST BYTES. When a table (or
3755 /// one of its blocks) cannot be read at all, what it said about its keys is
3756 /// gone with it, and older versions of those keys survive in other tables:
3757 /// a value it had overwritten, or a key its tombstone had deleted, becomes
3758 /// visible again. Neither setting changes that, because nothing left on
3759 /// disk distinguishes "this key was deleted" from "this key was simply
3760 /// never rewritten". Deleting the lost range instead would destroy intact
3761 /// data — a flushed table's key range routinely spans most of the keyspace
3762 /// while its sequence numbers sit above every older level, so one corrupt
3763 /// block would erase most of the tree, irreversibly, where the stale read
3764 /// leaves every byte in place. The lost ranges are therefore REPORTED
3765 /// rather than acted on; see
3766 /// [`RepairReport::lost_coverage`](crate::RepairReport::lost_coverage).
3767 ///
3768 /// # Errors
3769 ///
3770 /// Returns an error if the tables directory cannot be scanned or a transient
3771 /// I/O fault interrupts recovery (retryable), or if the rebuilt manifest
3772 /// cannot be durably installed. See
3773 /// [`repair_with_salvage`](Self::repair_with_salvage).
3774 pub fn repair_with_resurrection(
3775 &self,
3776 salvage: bool,
3777 allow_resurrection: bool,
3778 ) -> crate::Result<RepairReport> {
3779 repair_tree(self, salvage, allow_resurrection)
3780 }
3781
3782 /// Opens the tree, and when the open fails STRUCTURALLY, repairs per
3783 /// `policy` and opens again — the one-call recovery entry point for a
3784 /// caller that pairs the engine with an external write-ahead log.
3785 ///
3786 /// `Ok((tree, None))` is a healthy open. `Ok((tree, Some(report)))` is an
3787 /// open that succeeded only after a repair; the caller MUST then consult
3788 /// [`RepairReport::wal_replay_scope`] before its WAL replay, since the
3789 /// repair may have regressed persisted state below the log's trim
3790 /// watermark (see `docs/external-wal.md` § Replay after repair).
3791 ///
3792 /// Only failures that positively identify repairable on-disk structural
3793 /// damage engage the repair (see `is_repairable_structural`). Everything
3794 /// else is returned as-is: a TRANSIENT I/O failure (a repair would
3795 /// rebuild the manifest around files a healthy retry could still read), a
3796 /// held directory lock ([`Error::Locked`](crate::Error::Locked) — the
3797 /// repair would contend on the same lock), an UNSUPPORTED format version
3798 /// ([`Error::InvalidVersion`](crate::Error::InvalidVersion) — the store
3799 /// needs offline conversion or a matching binary, not a rebuild in the
3800 /// current format that would reject every table), a ROUTED tree's
3801 /// [`Error::Unrecoverable`](crate::Error::Unrecoverable) (route
3802 /// provenance is not persisted, so a missing routed table is
3803 /// indistinguishable from a route path change or an unmounted tier — a
3804 /// rebuild would omit every SST still on the old route), and every
3805 /// CONFIGURATION mismatch (wrong comparator, level route, dictionary, or
3806 /// encryption key) — repairing under a wrong configuration would rebuild,
3807 /// and could drop, perfectly healthy data.
3808 ///
3809 /// # Errors
3810 ///
3811 /// Propagates a transient-I/O or lock failure from the open and any
3812 /// repair failure. A failure of the POST-repair open surfaces as
3813 /// [`Error::RepairedButUnopened`](crate::Error::RepairedButUnopened),
3814 /// carrying the completed [`RepairReport`]: consume it from the error.
3815 ///
3816 /// The rebuilt snapshot holds its report until the caller acknowledges it
3817 /// with [`AbstractTree::acknowledge_repair_report`](crate::AbstractTree::acknowledge_repair_report),
3818 /// and every call before that answers `Some(report)` again, a healthy
3819 /// open included. A report is handed out at least once whatever ends the
3820 /// process between the hand-out and its use; one handed out again carries
3821 /// the same [`RepairReport::id`], so a caller that records the ids it
3822 /// applied skips it. Acknowledge only once the report's replay obligation
3823 /// is met.
3824 ///
3825 /// # Examples
3826 ///
3827 /// ```no_run
3828 /// use lsm_tree::{Config, RepairPolicy, SequenceNumberCounter};
3829 ///
3830 /// let config = Config::new(
3831 /// "/var/lib/mydb",
3832 /// SequenceNumberCounter::default(),
3833 /// SequenceNumberCounter::default(),
3834 /// );
3835 /// let (tree, repaired) = config.open_or_repair(RepairPolicy::default().salvage(true))?;
3836 /// if let Some(report) = repaired {
3837 /// println!("repaired; WAL must replay {:?}", report.wal_replay_scope());
3838 /// }
3839 /// # Ok::<_, lsm_tree::Error>(())
3840 /// ```
3841 pub fn open_or_repair(
3842 self,
3843 policy: RepairPolicy,
3844 ) -> crate::Result<(crate::AnyTree, Option<RepairReport>)> {
3845 let retry = self.clone();
3846 let path = self.path.clone();
3847 match self.open() {
3848 // A report not yet acknowledged rides in the snapshot this open
3849 // installed again.
3850 Ok(tree) => {
3851 let owed = owed_report(&tree, &path);
3852 Ok((tree, owed))
3853 }
3854 Err(e) if is_repairable_structural(&e) => {
3855 // The rebuilt snapshot owes the report, so a crash before it
3856 // reaches the caller leaves it for the next call.
3857 let report = repair_tree(&retry, policy.salvage, policy.allow_resurrection)?;
3858 match retry.open() {
3859 // The snapshot owes this same report (one id), but its
3860 // copy was written before the cleanup after the commit;
3861 // this call's own report also says what that cleanup did.
3862 Ok(tree) => Ok((tree, Some(report))),
3863 // The repair committed: the report leaves with the error,
3864 // and the snapshot that owes it hands it out again, with
3865 // the same id, to the call that opens it.
3866 Err(cause) => Err(crate::Error::RepairedButUnopened {
3867 report: alloc::boxed::Box::new(report),
3868 cause: alloc::boxed::Box::new(cause),
3869 }),
3870 }
3871 }
3872 Err(e) => Err(e),
3873 }
3874 }
3875}
3876
3877/// The [`RepairReport::id`] of the report encoded as `bytes`.
3878pub(crate) fn owed_report_id(bytes: &[u8]) -> u128 {
3879 report_marker::owed_id(bytes)
3880}
3881
3882/// The report `tree` owes until its caller acknowledges it.
3883fn owed_report(tree: &crate::AnyTree, path: &std::path::Path) -> Option<RepairReport> {
3884 let index = match tree {
3885 crate::AnyTree::Standard(tree) => tree,
3886 crate::AnyTree::Blob(tree) => &tree.index,
3887 };
3888 index
3889 .owed_repair_report()
3890 .map(|owed| report_marker::decode_owed(&owed, path))
3891}
3892
3893/// Whether an `open` failure positively identifies repairable ON-DISK
3894/// structural damage — the only class [`Config::open_or_repair`] may answer
3895/// with a repair.
3896///
3897/// Everything else propagates: transient I/O (a retry could still read the
3898/// files a repair would rebuild around), a held directory lock (the repair
3899/// would contend on the same lock), an UNSUPPORTED format version (an older
3900/// or future database has no live decoder here — it needs offline conversion
3901/// or a matching binary, and repair refuses it at the same format gate), and
3902/// every CONFIGURATION
3903/// mismatch — a wrong comparator, level route, zstd dictionary, a
3904/// standard-vs-blob tree-type mismatch ([`crate::Error::TreeTypeMismatch`]),
3905/// or an encryption key surfacing as a decrypt failure. Those are reversible by
3906/// fixing the call, while a repair under the wrong configuration rebuilds —
3907/// and can drop — perfectly healthy data (e.g. re-ordering SSTs under a
3908/// mistyped comparator, or salvage discarding every block it cannot decrypt
3909/// with the wrong key). Fail closed: an unlisted (including future) error is
3910/// NOT repaired.
3911fn is_repairable_structural(e: &crate::Error) -> bool {
3912 use crate::Error;
3913 match e {
3914 // Data-shape I/O kinds are structural: a missing file (the manifest
3915 // itself, or a file it names), malformed bytes (the manifest-loss
3916 // open surfaces "current missing but artifacts present" as
3917 // InvalidData), or a truncated read. Any other I/O kind is
3918 // environment, and transient kinds are retried by the caller.
3919 Error::Io(io) => matches!(
3920 io.kind(),
3921 crate::io::ErrorKind::NotFound
3922 | crate::io::ErrorKind::InvalidData
3923 | crate::io::ErrorKind::UnexpectedEof
3924 ),
3925 // `Unrecoverable` is deliberately ABSENT: it is the catch-all for
3926 // "cannot classify", the opposite of positive evidence of a structural
3927 // defect. Repairing on it rebuilds the manifest from whatever the
3928 // current configuration happens to scan, and since route provenance
3929 // is not persisted, a routed store reopened with its `level_routes`
3930 // omitted presents exactly that way, so the rebuild would drop every
3931 // SST on the unscanned tiers. An operator who has verified the
3932 // configuration can still invoke `repair` explicitly.
3933 Error::Decompress(_)
3934 | Error::Excised { .. }
3935 | Error::ChecksumMismatch { .. }
3936 | Error::HeaderCrcMismatch { .. }
3937 | Error::InvalidTag(_)
3938 | Error::InvalidTrailer
3939 | Error::InvalidHeader(_)
3940 | Error::DecompressedSizeTooLarge { .. }
3941 | Error::ManifestFrameChecksumMismatch { .. }
3942 | Error::ManifestFooterInvalid(_)
3943 | Error::ManifestSectionInvalid(_)
3944 | Error::TornManifestEditLog { .. }
3945 | Error::RangeTombstoneDecode { .. }
3946 | Error::PageEccUnrecoverable { .. }
3947 // A stored dictionary disagreeing with its own name is the bytes: the
3948 // name is their hash. The one configuration that reads the same way,
3949 // an encrypted file opened without its key, never gets here, because
3950 // `Tree::open` verifies the manifest under the configured provider
3951 // before it reads a dictionary. The repair sets the file aside, or
3952 // stops on a table that still needs it.
3953 | Error::ZstdDictCorrupt { .. } => true,
3954 _ => false,
3955 }
3956}
3957
3958/// Which repair capabilities [`Config::open_or_repair`] may engage.
3959///
3960/// Consulted when the open fails structurally. The default engages none of
3961/// them — the plain [`Config::repair`], which drops what it cannot recover
3962/// whole.
3963#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
3964pub struct RepairPolicy {
3965 /// Block-salvage SSTs / blob files that fail whole-file recovery instead
3966 /// of leaving them out (see [`Config::repair_with_salvage`]).
3967 pub salvage: bool,
3968 /// Keep ambiguous data (a lost restriction bound, an unauthenticated
3969 /// delete mask) instead of dropping it (see
3970 /// [`Config::repair_with_resurrection`]).
3971 pub allow_resurrection: bool,
3972}
3973
3974impl RepairPolicy {
3975 /// Enables block-level salvage (see [`Self::salvage`]).
3976 #[must_use]
3977 pub const fn salvage(mut self, enable: bool) -> Self {
3978 self.salvage = enable;
3979 self
3980 }
3981
3982 /// Enables resurrection of ambiguous data (see
3983 /// [`Self::allow_resurrection`]).
3984 #[must_use]
3985 pub const fn allow_resurrection(mut self, enable: bool) -> Self {
3986 self.allow_resurrection = enable;
3987 self
3988 }
3989}
3990
3991/// Completes the publication a previous repair committed but could not carry out.
3992///
3993/// A repair publishes by committing the manifest and only THEN swapping its
3994/// replacements onto their names and removing what the manifest no longer
3995/// references; a failure in that last step leaves the directory half-published
3996/// under a durable, correct manifest. Rebuilding from it instead would salvage
3997/// the damaged original again while the previous replacement is still there, so
3998/// one history would enter L0 twice — and duplicated merge operands are applied
3999/// twice on read.
4000///
4001/// The committed manifest is what resolves it. It is the tree's own authority
4002/// on which files count: an `open()` sweeps everything it does not name as an
4003/// orphan, so this applies the same rule before the scan runs — every table and
4004/// blob file it does not reference is removed. No state is carried between
4005/// runs; this is derived from the durable manifest alone.
4006///
4007/// A manifest that does not load cleanly is exactly the case repair exists for,
4008/// so it is not consulted at all then and the scan rebuilds from everything.
4009///
4010/// # Errors
4011///
4012/// Propagates a removal failure: leaving a superseded file in place is what
4013/// corrupts the rebuild, so the repair must fail rather than proceed past it.
4014/// What a CLEANLY-loading committed manifest hands the repair scan: the full
4015/// per-table records (checksum for temp-swap resolution, `global_seqno` so a
4016/// bulk-ingested survivor keeps its offset) and the committed RESTRICTIONS —
4017/// the authoritative bound for a tight-space-restricted survivor whose
4018/// sidecar was lost in the crash window the manifest exists to cover.
4019struct CommittedManifest {
4020 tables: crate::HashMap<TableId, crate::version::recovery::RecoveredTable>,
4021 restrictions: crate::HashMap<TableId, crate::UserKey>,
4022 /// The blob analogue of `restrictions`: each punched blob file's committed
4023 /// `[data_start, frontier)` boundary. Exact where the geometry walk only
4024 /// re-derives it — and where a backend that cannot attribute zeros to a
4025 /// hole re-derives it WRONG, reporting a punched file whole.
4026 blob_frontiers: crate::HashMap<crate::vlog::BlobFileId, u64>,
4027 /// The committed digest of each blob file's LIVE SUFFIX. Two copies of one
4028 /// id can both be self-consistent and still differ; only this says which
4029 /// one the manifest named, and re-stamping a fresh checksum over the other
4030 /// would leave existing SST handles resolving to the wrong generation.
4031 blob_checksums: crate::HashMap<crate::vlog::BlobFileId, crate::Checksum>,
4032}
4033
4034#[cfg(feature = "std")]
4035impl CommittedManifest {
4036 /// What this manifest says about `table_id`'s restriction.
4037 fn restriction_of(&self, table_id: TableId) -> ManifestRestriction {
4038 if !self.tables.contains_key(&table_id) {
4039 return ManifestRestriction::Unknown;
4040 }
4041 self.restrictions
4042 .get(&table_id)
4043 .map_or(ManifestRestriction::Unrestricted, |bound| {
4044 ManifestRestriction::Restricted(bound.clone())
4045 })
4046 }
4047}
4048
4049/// What a CLEANLY-loading committed manifest says about one scanned table's
4050/// restriction. It is the AUTHORITY in BOTH directions: the restriction is
4051/// committed BEFORE its `.restrict-bound` sidecar mirror is written, so the
4052/// manifest's bound is never the staler of the two; and a table it names
4053/// WITHOUT a restriction is genuinely unrestricted (a lifted restriction
4054/// drops out of the committed set), so a sidecar surviving there is stale
4055/// metadata that would hide a live prefix.
4056#[cfg(feature = "std")]
4057enum ManifestRestriction {
4058 /// The manifest names this table and restricts it to this exact bound.
4059 Restricted(crate::UserKey),
4060 /// The manifest names this table and does NOT restrict it: ignore any
4061 /// surviving sidecar. The punch geometry still runs — a table the
4062 /// manifest calls unrestricted has no committed reclaim, so zeroed
4063 /// blocks there are damage and must be classified as such, not opened
4064 /// over.
4065 Unrestricted,
4066 /// No clean manifest, or it does not name this table: the sidecar, then
4067 /// the punch geometry, decide.
4068 Unknown,
4069}
4070
4071#[cfg(feature = "std")]
4072fn sweep_superseded_by_committed_manifest(
4073 config: &Config,
4074) -> crate::Result<Option<CommittedManifest>> {
4075 let recovery = match crate::version::recovery::recover(
4076 &config.path,
4077 &*config.fs,
4078 crate::config::ManifestRecoveryMode::AbsoluteConsistency,
4079 config.encryption.clone(),
4080 ) {
4081 Ok(recovery) => recovery,
4082 // A one-shot read fault on an otherwise healthy manifest must NOT be
4083 // read as "no committed manifest exists": the repair would then scan
4084 // and republish every file, and with an unfinished compaction's inputs
4085 // and outputs both on disk the rebuilt L0 applies duplicate merge
4086 // operands — where the authoritative manifest would have named which
4087 // files are live. Propagate for a retry.
4088 Err(e) if is_environmental(&e) => return Err(e),
4089 // A manifest of another format is not damage: it is a store this
4090 // engine does not write. Rebuilding it here would convert it silently
4091 // and drop what only its manifest holds; the offline converter is
4092 // the one path from an older format.
4093 Err(e @ crate::Error::InvalidVersion(_)) => return Err(e),
4094 // A manifest that does not load cleanly is exactly the case repair
4095 // exists for: nothing committed to consult, the scan rebuilds from
4096 // everything.
4097 Err(_) => return Ok(None),
4098 };
4099
4100 // BEFORE anything is removed: a clean manifest's tree type is the
4101 // authority, and a run configured for the other type would rebuild the
4102 // manifest as that type — after which the CORRECT configuration fails
4103 // against a manifest this repair itself wrote. The scan cannot catch it
4104 // either: no surviving SST proves a Standard store is not a blob one (the
4105 // reverse is caught later, by a table that carries indirections). Refuse
4106 // while the store is still untouched.
4107 let requested = if config.kv_separation_opts.is_some() {
4108 TreeType::Blob
4109 } else {
4110 TreeType::Standard
4111 };
4112 if recovery.tree_type != requested {
4113 log::error!(
4114 "repair: the committed manifest describes a {:?} tree but this repair is \
4115 configured for {requested:?}; rebuilding would leave the store openable \
4116 only under the wrong configuration",
4117 recovery.tree_type,
4118 );
4119 return Err(crate::Error::TreeTypeMismatch {
4120 requested,
4121 actual: recovery.tree_type,
4122 });
4123 }
4124
4125 // The full per-table records, not just the id set: the checksum drives
4126 // temp-swap resolution below, and the `global_seqno` ingest offset is
4127 // what the caller's scan reuses for a healthy bulk-ingested table (a
4128 // clean manifest record makes that offset recoverable).
4129 let referenced_tables: crate::HashMap<TableId, crate::version::recovery::RecoveredTable> =
4130 recovery
4131 .table_ids
4132 .iter()
4133 .flatten()
4134 .flatten()
4135 .map(|t| (t.id, *t))
4136 .collect();
4137 for (table_base_folder, folder_fs) in config.all_tables_folders() {
4138 if !folder_fs.exists(&table_base_folder)? {
4139 continue;
4140 }
4141 for dirent in folder_fs.read_dir(&table_base_folder)? {
4142 if dirent.is_dir {
4143 continue;
4144 }
4145 // A replacement a previous run built. The manifest is the authority
4146 // on what it is — but the manifest names the id in BOTH crash cases
4147 // (before the commit its entry still describes the SOURCE), so the
4148 // entry's checksum is what decides: only a committed run recorded
4149 // the temp's digest, and swapping an uncommitted build in would
4150 // destroy the source the manifest names. Both answers still come
4151 // from the durable manifest alone.
4152 if let Some(id) = table_id_from_repair_tmp_name(&dirent.file_name) {
4153 let table_path = table_base_folder.join(id.to_string());
4154 let published = match referenced_tables.get(&id).map(|t| t.checksum) {
4155 Some(manifest_checksum) => repair_tmp_is_published(
4156 config,
4157 &folder_fs,
4158 &dirent.path,
4159 id,
4160 manifest_checksum,
4161 recovery.restrictions.get(&id),
4162 )?,
4163 None => false,
4164 };
4165 if published {
4166 commit_repair_tmp(
4167 &*folder_fs,
4168 &dirent.path,
4169 &table_path,
4170 config.sync_mode,
4171 recovery.restrictions.contains_key(&id),
4172 )?;
4173 log::info!(
4174 "repair: finished the pending swap of table {id} from a previous run",
4175 );
4176 } else {
4177 discard_unreferenced(&*folder_fs, &dirent.path, config.sync_mode)?;
4178 log::info!("repair: dropped an abandoned replacement for table {id}");
4179 }
4180 continue;
4181 }
4182 // Only files the scan would ADOPT as tables are swept here; a
4183 // foreign name is left to the scan's own classification.
4184 let Ok(id) = dirent.file_name.parse::<TableId>() else {
4185 continue;
4186 };
4187 if referenced_tables.contains_key(&id) {
4188 continue;
4189 }
4190 // Unreferenced bare-id files are removed even though one COULD be a
4191 // compaction input whose referenced output turns out corrupt below.
4192 // Deliberate, not an oversight: an output carries the SAME seqnos as
4193 // its inputs, so letting the scan adopt both would put one history
4194 // into L0 twice, and version collapse would then gather a key's
4195 // merge operands from both copies — applying each operand twice and
4196 // silently serving a wrong value. An honest loss (reported via
4197 // `lost_coverage`) beats that corruption; the sweep also only
4198 // applies the tree's own rule, since any successful open removes
4199 // the same orphans. Content a corrupt referenced file lost comes
4200 // back from a replica, a checkpoint plus journal replay, or a
4201 // backup — never from resurrected orphans.
4202 discard_unreferenced(&*folder_fs, &dirent.path, config.sync_mode)?;
4203 log::info!("repair: table {id} is superseded by the committed manifest; removed");
4204 }
4205 }
4206
4207 if config.kv_separation_opts.is_some() {
4208 let referenced_blobs: crate::HashSet<crate::vlog::BlobFileId> =
4209 recovery.blob_file_ids.iter().map(|(id, _)| *id).collect();
4210 let blobs_folder = config.path.join(crate::file::BLOBS_FOLDER);
4211 if config.fs.exists(&blobs_folder)? {
4212 for dirent in config.fs.read_dir(&blobs_folder)? {
4213 let Ok(id) = dirent.file_name.parse::<crate::vlog::BlobFileId>() else {
4214 continue;
4215 };
4216 if dirent.is_dir || referenced_blobs.contains(&id) {
4217 continue;
4218 }
4219 match config.fs.remove_file(&dirent.path) {
4220 Ok(()) => log::info!(
4221 "repair: blob file {id} is superseded by the committed manifest; removed",
4222 ),
4223 Err(e) if e.kind() == crate::io::ErrorKind::NotFound => {}
4224 Err(e) => return Err(e.into()),
4225 }
4226 }
4227 }
4228 }
4229 // The manifest loaded cleanly: hand the caller its referenced table
4230 // records, so an id it names that the directory scan never SEES (the
4231 // file is gone, not merely unreadable) is still reported as an
4232 // unscopable loss — and so a healthy bulk-ingested table's manifest-only
4233 // `global_seqno` offset is reused instead of failing closed. The
4234 // committed restrictions ride along: they are the authority for a
4235 // restricted survivor whose sidecar the crash took.
4236 Ok(Some(CommittedManifest {
4237 tables: referenced_tables,
4238 restrictions: recovery.restrictions,
4239 blob_frontiers: recovery.blob_restrictions,
4240 blob_checksums: recovery.blob_file_ids.iter().copied().collect(),
4241 }))
4242}
4243
4244/// Removes the fresh-id blob replacements an aborting run already published,
4245/// so a pre-commit exit leaves nothing behind that a retry would have to
4246/// re-create beside an orphan — under tight disk space exactly the sequence
4247/// that fails. `NotFound` is success (a concurrent sweep, or an in-place
4248/// rewrite that published no file); any other removal failure propagates to
4249/// the caller (the guard's `Drop` downgrades it to a log line, since the run
4250/// is already aborting with its own error).
4251fn remove_published_blob_replacements(
4252 config: &Config,
4253 blobs_folder: &std::path::Path,
4254 replacement_ids: impl IntoIterator<Item = crate::vlog::BlobFileId>,
4255) -> crate::Result<()> {
4256 let mut removed = false;
4257 for id in replacement_ids {
4258 match config.fs.remove_file(&blobs_folder.join(id.to_string())) {
4259 Ok(()) => removed = true,
4260 Err(e) if e.kind() == crate::io::ErrorKind::NotFound => {}
4261 Err(e) => return Err(e.into()),
4262 }
4263 }
4264 if removed {
4265 config
4266 .fs
4267 .sync_directory_with(blobs_folder, config.sync_mode)?;
4268 }
4269 Ok(())
4270}
4271
4272/// Every fresh-id blob replacement the running repair has PUBLISHED (renamed
4273/// onto its numeric name) before the manifest commit. Dropping the guard
4274/// while it is armed removes them: the manifest never adopted the files, and
4275/// each one left behind makes the retry salvage its original AGAIN beside
4276/// the orphan — under the tight disk space this recovery targets, exactly
4277/// the extra full copy that fails with ENOSPC. `Drop`-based so EVERY
4278/// pre-commit exit — an error `?`, a cancellation, an early return — unwinds
4279/// them uniformly; disarmed immediately before the manifest commit, from
4280/// which point the replacements are what the manifest names (a commit
4281/// FAILURE deliberately leaves them: the retry's unreferenced-file filter
4282/// removes what it does not adopt, and a post-rename commit error must not
4283/// delete files a switched manifest may already reference).
4284struct PublishedBlobReplacements<'a> {
4285 config: &'a Config,
4286 blobs_folder: PathBuf,
4287 ids: Vec<crate::vlog::BlobFileId>,
4288 armed: bool,
4289 /// The repair's first tree id: the replacements' descriptors are cached
4290 /// under ids from here on (see `HeldDescriptors`).
4291 first_tree_id: crate::tree::inner::TreeId,
4292}
4293
4294impl<'a> PublishedBlobReplacements<'a> {
4295 fn new(config: &'a Config, first_tree_id: crate::tree::inner::TreeId) -> Self {
4296 Self {
4297 config,
4298 blobs_folder: config.path.join(crate::file::BLOBS_FOLDER),
4299 ids: Vec::new(),
4300 armed: true,
4301 first_tree_id,
4302 }
4303 }
4304
4305 /// Records one replacement as published (its rename succeeded).
4306 fn publish(&mut self, id: crate::vlog::BlobFileId) {
4307 self.ids.push(id);
4308 }
4309
4310 /// The manifest commit is next: the replacements are about to become the
4311 /// files it names, so an exit from here on must not remove them.
4312 fn disarm(&mut self) {
4313 self.armed = false;
4314 }
4315}
4316
4317/// What the disk says about a failed `persist_version`: was the rebuilt
4318/// snapshot installed (renamed to its name) before the error?
4319enum InstallProbe {
4320 /// The rebuilt snapshot's name exists: it is, or may become, installed.
4321 Installed,
4322 /// The name is ABSENT (`NotFound`): the install did not happen. Absence
4323 /// is conclusive: the rebuilt id is above every manifest file on disk, so
4324 /// only the persist's rename makes the name, and only the persist's
4325 /// withdrawal removes it after, which the persist reports as
4326 /// [`crate::Error::InstallOutcomeUnknown`] unless it made the removal
4327 /// durable.
4328 NotInstalled,
4329 /// The name's existence cannot be told (a transient or permission failure
4330 /// of the probe): NO conclusion. The caller must fail SAFE: treating
4331 /// this as "not installed" would delete files a possibly-published
4332 /// manifest references.
4333 Inconclusive,
4334}
4335
4336/// The post-failure probe deciding if a failed `persist_version` died before
4337/// or after it installed the snapshot.
4338fn probe_install(
4339 fs: &dyn crate::fs::Fs,
4340 folder: &std::path::Path,
4341 version_id: u64,
4342) -> InstallProbe {
4343 match fs.metadata(&folder.join(format!("v{version_id}"))) {
4344 Ok(_) => InstallProbe::Installed,
4345 Err(e) if e.kind() == crate::io::ErrorKind::NotFound => InstallProbe::NotInstalled,
4346 Err(_) => InstallProbe::Inconclusive,
4347 }
4348}
4349
4350impl Drop for PublishedBlobReplacements<'_> {
4351 fn drop(&mut self) {
4352 if !self.armed || self.ids.is_empty() {
4353 return;
4354 }
4355 // The repair is ending: its cached descriptors, the replacements'
4356 // among them, close before the files go, or a backend that refuses to
4357 // remove an open file keeps them. The repair's own guard sweeps the
4358 // same ids later, and only after this one has run.
4359 if let Some(cache) = &self.config.descriptor_table {
4360 cache.remove_trees_from(self.first_tree_id);
4361 }
4362 // Best-effort: the run is already aborting with its own error, and a
4363 // survivor is an unreferenced file the next open's orphan sweep (or
4364 // the retry's unreferenced-file filter) removes.
4365 if let Err(e) =
4366 remove_published_blob_replacements(self.config, &self.blobs_folder, self.ids.drain(..))
4367 {
4368 log::warn!(
4369 "repair: could not remove a published blob replacement while \
4370 aborting ({e}); the next open's orphan sweep removes it",
4371 );
4372 }
4373 }
4374}
4375
4376/// Surfaces a caller's cooperative cancellation
4377/// ([`RecoveryProgress::request_cancel`](crate::RecoveryProgress::request_cancel))
4378/// as [`Error::Cancelled`](crate::Error::Cancelled). Consulted at file
4379/// boundaries, and ONLY before the manifest commit: the pre-commit scan is
4380/// read-only, so an abort there leaves the directory exactly as a retry
4381/// expects, while aborting the post-commit swaps or removals would leave work
4382/// the next open must redo anyway.
4383fn check_cancel(config: &Config) -> crate::Result<()> {
4384 if let Some(p) = &config.recovery_progress
4385 && p.is_cancel_requested()
4386 {
4387 return Err(crate::Error::Cancelled);
4388 }
4389 Ok(())
4390}
4391
4392/// Publishes the repair's byte total (every file in the table folders, plus
4393/// the blob folder for a KV-separated tree) to the progress handle, if any.
4394/// Best-effort on purpose: a file that cannot be stat'ed is left out of the
4395/// total rather than failing the repair over a display number — the scan
4396/// itself will classify it.
4397fn publish_recovery_bytes_total(config: &Config) {
4398 let Some(progress) = &config.recovery_progress else {
4399 return;
4400 };
4401 let mut total: u64 = 0;
4402 let mut add_folder = |folder: &std::path::Path, fs: &dyn crate::fs::Fs| {
4403 let Ok(true) = fs.exists(folder) else {
4404 return;
4405 };
4406 let Ok(dirents) = fs.read_dir(folder) else {
4407 return;
4408 };
4409 for dirent in dirents {
4410 // Same skips as the scan loops, so `bytes_processed` can reach
4411 // `bytes_total` exactly.
4412 if dirent.is_dir {
4413 continue;
4414 }
4415 if let Ok(meta) = fs.metadata(&dirent.path) {
4416 // Display-only sum: clamping at u64::MAX merely freezes the
4417 // percentage, while a checked overflow would fail the repair
4418 // over a progress number.
4419 total = total.saturating_add(meta.len);
4420 }
4421 }
4422 };
4423 for (folder, fs) in config.all_tables_folders() {
4424 add_folder(&folder, &*fs);
4425 }
4426 if config.kv_separation_opts.is_some() {
4427 add_folder(&config.path.join(crate::file::BLOBS_FOLDER), &*config.fs);
4428 }
4429 progress.set_bytes_total(total);
4430}
4431
4432fn repair_tree(
4433 config: &Config,
4434 salvage: bool,
4435 allow_resurrection: bool,
4436) -> crate::Result<RepairReport> {
4437 // Hold the cross-process directory lock for the whole repair: it writes and
4438 // installs a fresh snapshot and sweeps `edits-*` in place, so a
4439 // concurrent open / repair of the same directory would corrupt the manifest.
4440 // A second acquirer fails fast with `Error::Locked`. Dropped at function
4441 // return, releasing the lock. The directory is expected to exist (repair
4442 // operates on an existing tree).
4443 #[cfg(feature = "std")]
4444 let _directory_lock =
4445 crate::config::acquire_directory_lock(&*config.fs, &config.path, config.directory_lock)?;
4446
4447 // A repair opens every SST it finds, so it needs the tree's dictionaries
4448 // exactly as an open does — and it does NOT go through `Tree::open`, which
4449 // is where they would otherwise be loaded. Without this every
4450 // dictionary-compressed table is graded unreadable and the rebuilt manifest
4451 // leaves the tree's data behind. Under the directory lock, since it reads
4452 // the tree's own folder, and on a COPY, so the caller's config keeps the
4453 // registry it had (a `Config` is cloned per tree, and they must not share).
4454 //
4455 // A dictionary whose bytes no longer hash to its name is left out rather
4456 // than failing the load: the repair is for exactly that tree. A table that
4457 // needs it then fails like one whose dictionary was never supplied, which
4458 // aborts rather than drops it, so a re-run with the right dictionary still
4459 // recovers it.
4460 #[cfg(zstd_any)]
4461 let (owned_config, damaged_dictionaries) = {
4462 let mut owned = config.clone();
4463 let damaged = owned.install_own_zstd_dictionaries_skipping_damaged()?;
4464 (owned, damaged)
4465 };
4466 #[cfg(zstd_any)]
4467 let config = &owned_config;
4468
4469 // The tables the repair holds keep their files in the configured
4470 // descriptor cache, as an open tree does, each under a tree id of its own
4471 // handed out from here on; every id from this one is swept out of the
4472 // cache before the post-commit swaps (see `held_recover_params`).
4473 let first_tree_id = crate::tree::inner::get_next_tree_id();
4474 // However the repair ends (an abort, a cancellation, a failed commit),
4475 // the descriptors it cached close with it: left open, a temp it built
4476 // could not be removed by the retry on a backend that refuses to unlink
4477 // an open file.
4478 let _held = HeldDescriptors {
4479 cache: config.descriptor_table.clone(),
4480 first: first_tree_id,
4481 };
4482
4483 if let Some(p) = &config.recovery_progress {
4484 p.set_phase(crate::RecoveryPhase::PendingSwaps);
4485 }
4486 check_cancel(config)?;
4487
4488 // Finish the previous run's outstanding cleanup first. See
4489 // `sweep_superseded_by_committed_manifest`. When the manifest itself
4490 // loads cleanly (repair was entered over a MISSING file, not a lost
4491 // manifest), its referenced table records are kept: an id it names that
4492 // the directory scan never sees has no directory entry to report
4493 // through, and losing it silently would let `wal_replay_scope()` answer
4494 // `TailOnly` over lost persisted data. The records also carry each
4495 // table's `global_seqno`, so a healthy bulk-ingested SST keeps its
4496 // manifest-only ingest offset instead of being failed closed.
4497 #[cfg(feature = "std")]
4498 let manifest_referenced: Option<CommittedManifest> =
4499 sweep_superseded_by_committed_manifest(config)?;
4500 #[cfg(not(feature = "std"))]
4501 let manifest_referenced: Option<CommittedManifest> = None;
4502
4503 // An earlier repair whose report never reached the caller: the installed
4504 // snapshot still owes it. This rebuild replaces that snapshot, so the
4505 // earlier losses travel in its report. Only that one section is read, so
4506 // damage elsewhere in the snapshot does not hide it. A snapshot too
4507 // damaged to say whether it owes one may: that replay is owed unscoped,
4508 // since leaving it out would lose it for good. A transient failure
4509 // propagates for the retry.
4510 let outstanding = crate::version::recovery::owed_repair_report(
4511 &config.path,
4512 &*config.fs,
4513 config.encryption.clone(),
4514 )?
4515 .map(|owed| match owed {
4516 crate::version::recovery::OwedReport::Readable(bytes) => {
4517 report_marker::decode_owed(&bytes, &config.path)
4518 }
4519 crate::version::recovery::OwedReport::Unreadable(id) => {
4520 report_marker::unreadable_owed(&config.path, id)
4521 }
4522 });
4523
4524 // Byte totals for the progress percentage, from an upfront listing —
4525 // before the scan phase starts, so `bytes_processed / bytes_total` is
4526 // meaningful from the first processed file.
4527 publish_recovery_bytes_total(config);
4528 if let Some(p) = &config.recovery_progress {
4529 p.set_phase(crate::RecoveryPhase::ScanningTables);
4530 }
4531
4532 // Phase 1: read every table folder and decide, per file, what it is and
4533 // whether it can be published. Read-only on disk — nothing is removed or
4534 // swapped until the rebuilt manifest is durable, so an abort here leaves
4535 // the directory exactly as the retry expects it.
4536 let scan = scan_table_folders(
4537 config,
4538 salvage,
4539 allow_resurrection,
4540 manifest_referenced.as_ref(),
4541 )?;
4542
4543 // Phase 2: turn what the scan found into a manifest, commit it, and carry
4544 // out the removals and swaps that commit authorizes. The directory lock is
4545 // held by THIS frame for the whole of it.
4546 let rebuilt = rebuild_from_scan(
4547 config,
4548 allow_resurrection,
4549 manifest_referenced,
4550 scan,
4551 first_tree_id,
4552 outstanding,
4553 #[cfg(zstd_any)]
4554 &damaged_dictionaries,
4555 );
4556 #[cfg(zstd_any)]
4557 let rebuilt = set_aside_damaged_dictionaries(config, damaged_dictionaries, rebuilt);
4558 rebuilt
4559}
4560
4561/// Moves every damaged dictionary still under its name aside, now that the
4562/// rebuilt manifest is durable.
4563///
4564/// Left under its name, a damaged dictionary would fail the open this repair
4565/// was run for. One the rebuild rewrote from a supplied copy reads intact
4566/// again and stays. The report already names each one moved here: the rebuild
4567/// records them before it writes the report's copy, which an open hands out
4568/// when the run ends after these moves and before the report reaches the
4569/// caller.
4570///
4571/// # Errors
4572///
4573/// A failure here, like one inside the rebuild's own post-commit steps, comes
4574/// back as [`crate::Error::RepairedButUnopened`] carrying the report: the
4575/// manifest is durable, so the repair happened, and a retry would find nothing
4576/// left to repair and answer with no report at all. Every dictionary is still
4577/// tried; the first failure is the one returned. A rebuild that failed before
4578/// its commit is passed through untouched, with nothing set aside.
4579#[cfg(zstd_any)]
4580fn set_aside_damaged_dictionaries(
4581 config: &Config,
4582 damaged: Vec<crate::file::DictId>,
4583 rebuilt: crate::Result<RepairReport>,
4584) -> crate::Result<RepairReport> {
4585 let (mut report, mut post_commit_error) = match rebuilt {
4586 Ok(report) => (report, None),
4587 Err(crate::Error::RepairedButUnopened { report, cause }) => (*report, Some(*cause)),
4588 Err(e) => return Err(e),
4589 };
4590 let folder = config.path.join(crate::file::DICTS_FOLDER);
4591 for id in damaged {
4592 let moved = match crate::dicts::set_aside_if_damaged(
4593 &*config.fs,
4594 &folder,
4595 id,
4596 config.encryption.as_deref(),
4597 config.sync_mode,
4598 ) {
4599 Ok(moved) => moved.is_some(),
4600 Err(e) => {
4601 post_commit_error.get_or_insert(e);
4602 false
4603 }
4604 };
4605 // The report's entry, written before the move, says now what it did.
4606 let aside = crate::dicts::damaged_path(&folder, id);
4607 if let Some(entry) = report
4608 .damaged_dictionaries
4609 .iter_mut()
4610 .find(|(path, _)| *path == aside)
4611 {
4612 entry.1 = if moved {
4613 format!("dictionary {id} failed its integrity check; moved aside")
4614 } else {
4615 format!("dictionary {id} failed its integrity check; NOT moved aside")
4616 };
4617 }
4618 }
4619 match post_commit_error {
4620 Some(cause) => Err(crate::Error::RepairedButUnopened {
4621 report: Box::new(report),
4622 cause: Box::new(cause),
4623 }),
4624 None => Ok(report),
4625 }
4626}
4627
4628/// Everything [`scan_table_folders`] learned from the table folders, and the
4629/// only channel between the scan and the rebuild.
4630///
4631/// The scan's own bookkeeping ends here: a name it classified, a file it could
4632/// not read, a replacement it built. Nothing else it touched survives the
4633/// phase boundary, so the rebuild cannot accidentally depend on a scan
4634/// intermediate.
4635#[cfg(feature = "std")]
4636struct TableScan {
4637 /// Best recovered copy per table id (a complete recovery beats a lossy
4638 /// salvage).
4639 recovered_by_id: crate::HashMap<TableId, TableCandidate>,
4640 /// Files the rebuild must report as damaged.
4641 unreadable_files: Vec<(PathBuf, String)>,
4642 /// Damaged files that cost NO coverage: a duplicate of an id whose
4643 /// complete copy is retained. They belong in the corruption channel — an
4644 /// operator watching for a failing disk must see them — but not in the
4645 /// loss accounting, where an unscopable entry would tell an external WAL
4646 /// to reconcile the whole keyspace over a tree that lost nothing.
4647 redundant_unreadable: crate::HashSet<PathBuf>,
4648 /// HEALTHY files the rebuild deliberately leaves out.
4649 excluded_files: Vec<(PathBuf, String)>,
4650 /// What each scanned table covers, captured while its metadata was
4651 /// readable.
4652 coverage_by_path: crate::HashMap<PathBuf, (UserKey, UserKey, Option<SeqNo>)>,
4653 /// Files to remove once the rebuilt manifest is durable.
4654 discard_after_commit: Vec<(Arc<dyn crate::fs::Fs>, PathBuf, String)>,
4655 /// Replacements to swap onto their final names once it is.
4656 swap_after_commit: Vec<(Arc<dyn crate::fs::Fs>, PathBuf, PathBuf, bool)>,
4657 /// Sidecars found beside tables, judged against the FINAL rebuilt set.
4658 live_sidecars: Vec<(TableId, PathBuf, Arc<dyn crate::fs::Fs>)>,
4659 /// Every table id the scan actually SAW on disk, joined against the
4660 /// committed manifest to report ids whose file is gone entirely.
4661 scanned_table_ids: crate::HashSet<TableId>,
4662}
4663
4664/// The tables that opened in one folder: each id with the ids below it its
4665/// folder was still writing when it was finished.
4666#[cfg(feature = "std")]
4667type OpenedTables = crate::HashMap<PathBuf, Vec<(TableId, Vec<TableId>)>>;
4668
4669/// Records that `table` opened as a table in `folder`, so an empty file
4670/// there below it, written by the time it was finished, is judged a table
4671/// that lost its bytes.
4672#[cfg(feature = "std")]
4673fn note_opened(opened: &mut OpenedTables, folder: &std::path::Path, table: &Table) {
4674 opened
4675 .entry(folder.to_path_buf())
4676 .or_default()
4677 .push((table.id(), table.metadata.writing_below.clone()));
4678}
4679
4680/// Whether one of `tables` (ids with what their folder was still writing
4681/// below them) was finished after `id`'s file had its table written: one
4682/// above it that does not list it.
4683#[cfg(feature = "std")]
4684fn written_before_one_of<'a>(
4685 id: TableId,
4686 mut tables: impl Iterator<Item = (TableId, &'a [TableId])>,
4687) -> bool {
4688 tables.any(|(table, writing_below)| id < table && !writing_below.contains(&id))
4689}
4690
4691/// Phase 1 of [`repair_tree`]: classify every entry of every table folder and
4692/// recover what can be recovered. See [`TableScan`] for what it hands on.
4693///
4694/// # Errors
4695///
4696/// Propagates an environmental failure (the retry re-reads the same bytes) and
4697/// any failure that must abort the whole repair rather than grade one file.
4698#[cfg(feature = "std")]
4699fn scan_table_folders(
4700 config: &Config,
4701 salvage: bool,
4702 allow_resurrection: bool,
4703 manifest_referenced: Option<&CommittedManifest>,
4704) -> crate::Result<TableScan> {
4705 let mut scanned_table_ids: crate::HashSet<TableId> = crate::HashSet::default();
4706
4707 // Best recovered copy per table id: a complete recovery beats a lossy salvage,
4708 // so a duplicate id across aliased / routed table folders keeps only the best
4709 // (and is never added to two L0 runs). See `keep_best_candidate`.
4710 let mut recovered_by_id: crate::HashMap<TableId, TableCandidate> = crate::HashMap::default();
4711 let mut unreadable_files: Vec<(PathBuf, String)> = Vec::new();
4712 let mut redundant_unreadable: crate::HashSet<PathBuf> = crate::HashSet::default();
4713 // HEALTHY files the rebuild deliberately leaves out (valid duplicates,
4714 // lineage-redundant outputs and inputs): reported in the report's
4715 // `excluded_files`, never in the unreadable counts — they opened and
4716 // verified fine, and counting them as unreadable fires corruption
4717 // alerts over an intact store.
4718 let mut excluded_files: Vec<(PathBuf, String)> = Vec::new();
4719 // What each scanned table covers, captured while its metadata was readable.
4720 // Joined against the exclusions at the end to report the coverage the
4721 // rebuilt manifest lost.
4722 let mut coverage_by_path: crate::HashMap<PathBuf, (UserKey, UserKey, Option<SeqNo>)> =
4723 crate::HashMap::default();
4724 // Files the rebuilt manifest will not name: a foreign name, a duplicate id,
4725 // a table no bound can make safe, a source a salvaged copy supersedes. They
4726 // are removed AFTER the commit and not one moment earlier — the scan is
4727 // read-only precisely so a crash leaves the directory as the retry expects
4728 // to find it, and the retry then derives everything again from the same
4729 // bytes. Each entry carries the backend that owns the file: a per-level
4730 // route stores its tables through an `Fs` the primary one cannot see.
4731 let mut discard_after_commit: Vec<(Arc<dyn crate::fs::Fs>, PathBuf, String)> = Vec::new();
4732 // Replacements built at `{id}.repair-tmp`, swapped onto `{id}` once the
4733 // manifest that adopts them is durable. Same ordering rule as the removals,
4734 // for the same reason: before the commit the source is still the only copy
4735 // of its rows and is what the manifest names.
4736 // (fs, temp path, final path, replacement is restricted): the last flag
4737 // rides along so a retried swap can tell an already-moved replacement
4738 // sidecar from a stale source one (see `commit_repair_tmp`).
4739 let mut swap_after_commit: Vec<(Arc<dyn crate::fs::Fs>, PathBuf, PathBuf, bool)> = Vec::new();
4740
4741 // Live sidecars ({id}.heal-attest, {id}.restrict-bound) found by the scan.
4742 // Judged against the FINAL rebuilt set after the dedup: one whose table
4743 // survived is preserved, an orphan is removed post-commit.
4744 let mut live_sidecars: Vec<(TableId, PathBuf, Arc<dyn crate::fs::Fs>)> = Vec::new();
4745
4746 // Empty files under table names no committed manifest names:
4747 // `(id, folder, path, fs)`, judged against the tables their folder
4748 // recovered once the scan is done.
4749 let mut empty_unnamed: Vec<(TableId, PathBuf, PathBuf, Arc<dyn crate::fs::Fs>)> = Vec::new();
4750 // The files that opened as tables, per folder, whether the rebuild then
4751 // keeps them, sets them aside or leaves them out as duplicates: tables
4752 // written there either way, each telling which files below it were
4753 // still being written when it was finished.
4754 let mut opened_tables: OpenedTables = crate::HashMap::default();
4755
4756 for (table_base_folder, folder_fs) in config.all_tables_folders() {
4757 if !folder_fs.exists(&table_base_folder)? {
4758 continue;
4759 }
4760
4761 // ORDER the entries before recovering: `read_dir` order is
4762 // FS-dependent, and duplicate-id resolution keeps the FIRST complete
4763 // copy — so an unordered scan would rebuild different contents from the
4764 // same directory across runs. Per id, the writer's own `id.to_string()`
4765 // spelling is the canonical file and sorts first, so a foreign alternate
4766 // spelling (`01` for id 1) can never displace it. Mirrors the blob-file
4767 // scan.
4768 let mut dirents = folder_fs.read_dir(&table_base_folder)?;
4769 dirents.sort_by(|a, b| {
4770 // Group every name that belongs to one id together, and inside a
4771 // group put the abandoned `{id}.repair-tmp` FIRST: this scan
4772 // removes it (no manifest names it — one that did was swapped in
4773 // before the scan started), and a salvage of `{id}` builds its
4774 // replacement under exactly that name. Visiting the source first
4775 // would find the destination occupied, fail the salvage with
4776 // `AlreadyExists` — not an environmental error — and condemn a
4777 // source whose salvage would otherwise have succeeded.
4778 let key = |e: &crate::fs::FsDirEntry| {
4779 let numeric = e.file_name.parse::<TableId>().ok();
4780 let temp = table_id_from_repair_tmp_name(&e.file_name);
4781 let id = numeric.or(temp);
4782 // Within an id: the temp, then the writer's own canonical
4783 // spelling, then any foreign alternate spelling (`01` for id
4784 // 1) — which can therefore never displace the canonical file.
4785 let rank = if temp.is_some() {
4786 0
4787 } else if numeric.is_some_and(|id| e.file_name == id.to_string()) {
4788 1
4789 } else {
4790 2
4791 };
4792 (id.is_none(), id.unwrap_or(0), rank)
4793 };
4794 key(a)
4795 .cmp(&key(b))
4796 .then_with(|| a.file_name.cmp(&b.file_name))
4797 });
4798
4799 'dirent: for dirent in dirents {
4800 let crate::fs::FsDirEntry {
4801 path: table_path,
4802 file_name,
4803 is_dir,
4804 } = dirent;
4805
4806 if is_dir {
4807 continue;
4808 }
4809
4810 // Pre-commit file boundary: safe to abort (the scan is read-only),
4811 // and the file counts toward the progress percentage as soon as
4812 // its processing starts. Best-effort stat, like the total.
4813 check_cancel(config)?;
4814 if let Some(p) = &config.recovery_progress
4815 && let Ok(meta) = folder_fs.metadata(&table_path)
4816 {
4817 p.add_bytes_processed(meta.len);
4818 }
4819
4820 // One grammar decides what each name IS (`TableDirEntry`, shared
4821 // with `Tree::open`'s sweep so the two can never disagree on
4822 // ownership); this match is the repair scan's POLICY for each kind.
4823 use crate::file::TableDirEntry;
4824 let table_id = match TableDirEntry::classify(&file_name) {
4825 // LIVE sidecar artifacts are not table files, and repair must
4826 // not remove one whose table survives: `Tree::open` PRESERVES
4827 // a live `{id}.heal-attest` (the next scrub reconciles a
4828 // crashed digest refresh through it), and the
4829 // `.restrict-bound` bound is read FOR its SST via
4830 // `restrict_bound::read`. Whether a sidecar is live is
4831 // decided AGAINST THE REBUILT SET after the dedup below —
4832 // one whose table did not survive is an orphan the repair
4833 // itself removes, because leaving it hands the next open an
4834 // unconditional sweep whose refused unlink would turn this
4835 // run's success into an unopenable tree.
4836 TableDirEntry::HealAttest(id) | TableDirEntry::RestrictBound(id) => {
4837 live_sidecars.push((id, table_path, Arc::clone(&folder_fs)));
4838 continue;
4839 }
4840 // The disposable `.tmp` shapes are crash leftovers no state
4841 // references; the open sweeps them unconditionally, so repair
4842 // removes them itself (post-commit) for the same reason.
4843 TableDirEntry::HealAttestTmp(_)
4844 | TableDirEntry::HealTmp(_)
4845 | TableDirEntry::RestrictBoundTmp(_) => {
4846 discard_after_commit.push((
4847 Arc::clone(&folder_fs),
4848 table_path,
4849 "disposable crashed-heal artifact; the next open would \
4850 sweep it and fail on a refused removal"
4851 .to_string(),
4852 ));
4853 continue;
4854 }
4855 // An abandoned replacement's restriction sidecar: its temp is
4856 // removed by the `RepairTmp` arm (which also takes the
4857 // companion, so this entry is usually already gone — the
4858 // post-commit removal tolerates that), and a swept temp's
4859 // survivor is an orphan.
4860 TableDirEntry::RepairTmpCompanion(_) => {
4861 discard_after_commit.push((
4862 Arc::clone(&folder_fs),
4863 table_path,
4864 "restriction sidecar of an abandoned repair replacement".to_string(),
4865 ));
4866 continue;
4867 }
4868 // `{id}.repair-tmp` is a replacement a previous repair was
4869 // building and never published: no manifest names it (one that
4870 // did was swapped in before this scan started), and its source
4871 // is whatever this scan finds under `{id}`. It is therefore
4872 // garbage, and it is removed NOW rather than after the commit —
4873 // this run needs the name free to build its own replacement.
4874 // Removing it cannot lose rows: every row it holds came from a
4875 // file this scan is about to read.
4876 TableDirEntry::RepairTmp(_) => {
4877 // The removal also takes the temp's restriction
4878 // companion, whose own directory entry is visited LATER
4879 // in this scan — by then its stat returns NotFound and
4880 // the byte-progress prologue skips it, leaving a
4881 // successful repair short of 100%. Credit its bytes now,
4882 // before they disappear.
4883 if let Some(p) = &config.recovery_progress
4884 && let Ok(meta) =
4885 folder_fs.metadata(&crate::restrict_bound::sidecar_path(&table_path))
4886 {
4887 p.add_bytes_processed(meta.len);
4888 }
4889 discard_unreferenced(&*folder_fs, &table_path, config.sync_mode)?;
4890 log::warn!(
4891 "repair: dropped an abandoned replacement {}",
4892 table_path.display(),
4893 );
4894 continue;
4895 }
4896 // Not a shape the engine names, so not part of the inventory a
4897 // repair rebuilds: left exactly where it is. Removing it to
4898 // make the tree openable would be destroying an operator's file
4899 // to fix a problem the scanner invented, and the open no longer
4900 // has that problem.
4901 TableDirEntry::Foreign => {
4902 log::debug!(
4903 "repair: ignoring {} in the tables folder: not an engine file",
4904 table_path.display(),
4905 );
4906 continue;
4907 }
4908 TableDirEntry::Table(id) => id,
4909 };
4910 // An empty file under a table name that no committed manifest
4911 // names is judged once every table of its folder is recovered
4912 // (see `empty_unnamed`). A file whose length cannot be read is
4913 // graded as any table file is.
4914 let named = manifest_referenced
4915 .as_ref()
4916 .is_some_and(|m| m.tables.contains_key(&table_id));
4917 if !named
4918 && folder_fs
4919 .metadata(&table_path)
4920 .is_ok_and(|meta| meta.len == 0)
4921 {
4922 empty_unnamed.push((
4923 table_id,
4924 table_base_folder.clone(),
4925 table_path,
4926 Arc::clone(&folder_fs),
4927 ));
4928 continue;
4929 }
4930 scanned_table_ids.insert(table_id);
4931
4932 if let Some(p) = &config.recovery_progress {
4933 p.table_discovered();
4934 }
4935
4936 // A CLEAN manifest record for this id carries the table's
4937 // `global_seqno` and L0 recency: table files are immutable once
4938 // published and ids are never reused, so the record describes THIS
4939 // logical table even when its bytes have since been damaged.
4940 // Reusing it keeps a healthy bulk-ingested SST (and its real
4941 // sequence and L0 position) where the manifest-loss rule would have
4942 // to fail closed.
4943 let manifest_position: Option<ManifestPosition> = manifest_referenced
4944 .as_ref()
4945 .and_then(|m| m.tables.get(&table_id))
4946 .map(ManifestPosition::of_record);
4947 // What the committed manifest says about this id's restriction —
4948 // authoritative in both directions (see `ManifestRestriction`).
4949 let manifest_restriction = manifest_referenced
4950 .as_ref()
4951 .map_or(ManifestRestriction::Unknown, |m| m.restriction_of(table_id));
4952
4953 // This file's whole-file digest, computed ONCE: the duplicate
4954 // verdict below and the recovery further down both need it. A
4955 // RESTRICTED table's manifest entry digests only its live suffix,
4956 // which this never reproduces, so `matches_manifest` stays false
4957 // (uncomparable) rather than reading as a mismatch.
4958 let own_digest = match compute_table_checksum(&*folder_fs, &table_path) {
4959 Ok(c) => Ok(crate::Checksum::from_raw(c)),
4960 // A fault in the ENVIRONMENT is not evidence about these bytes.
4961 Err(e) if is_environmental(&e) => return Err(e),
4962 Err(e) => Err(e),
4963 };
4964 let committed_digest = manifest_referenced
4965 .as_ref()
4966 .and_then(|m| m.tables.get(&table_id))
4967 .map(|t| t.checksum);
4968 let matches_manifest = match (&own_digest, committed_digest) {
4969 (Ok(d), Some(committed)) => {
4970 match trustworthy_restriction_bound(
4971 config,
4972 &*folder_fs,
4973 &table_path,
4974 table_id,
4975 &manifest_restriction,
4976 )? {
4977 // A RESTRICTED entry digests only the LIVE SUFFIX, so
4978 // the whole-file hash can never equal it. Comparing them
4979 // would mark BOTH copies of a restricted id unmatched
4980 // and hand the choice back to scan order, the very
4981 // thing the digest is here to prevent. Reproduce the
4982 // suffix digest the restricted view records instead.
4983 Some(bound) => restricted_suffix_digest(
4984 config,
4985 &folder_fs,
4986 &table_path,
4987 table_id,
4988 &bound,
4989 )?
4990 .is_some_and(|suffix| suffix == committed),
4991 None => *d == committed,
4992 }
4993 }
4994 _ => false,
4995 };
4996
4997 // Skip a duplicate id ONLY when we already hold a COMPLETE copy — a
4998 // duplicate cannot improve on it. A previously-seen LOSSY salvage does
4999 // NOT skip: this copy is still evaluated and may supersede it.
5000 //
5001 // "Complete" is not enough on its own: two complete copies can hold
5002 // different generations, and the manifest says which one this id
5003 // is. A copy that reproduces its digest is evaluated even against a
5004 // complete incumbent, so it can displace one that does not.
5005 if let Some(existing) = recovered_by_id
5006 .get(&table_id)
5007 .filter(|c| c.fidelity.is_complete())
5008 .filter(|c| c.matches_manifest || !matches_manifest)
5009 {
5010 // If this path physically ALIASES the retained copy (a symlink /
5011 // junction / case-insensitive alias resolving to the same directory
5012 // entry, e.g. two configured folders pointing at one location), it
5013 // is the SAME file, not a genuine duplicate. Removing it would
5014 // destroy the kept copy and leave the manifest referencing a
5015 // missing SST, so skip it IN PLACE.
5016 if same_physical_file(&*folder_fs, &table_path, &*existing.fs, &existing.path)? {
5017 continue;
5018 }
5019 // A genuine duplicate: removed after the commit so recovery
5020 // cannot later resolve it instead of the kept copy (the manifest
5021 // records only id + checksum, not a path). It cannot improve the
5022 // rebuild, but the two report channels make DIFFERENT claims —
5023 // `excluded_files` promises a file that opened and verified,
5024 // `unreadable_files` is the corruption signal an operator
5025 // watches. Inferring health from the RETAINED copy's fidelity
5026 // would file a rotting duplicate as healthy and leave
5027 // `unreadable == 0` over a failing disk, so verify this copy on
5028 // its own bytes before choosing.
5029 let verdict = match own_digest {
5030 Ok(digest) => Table::recover(repair_recover_params(
5031 config,
5032 table_path.clone(),
5033 digest,
5034 table_id,
5035 folder_fs.clone(),
5036 manifest_position,
5037 ))
5038 // It opened as a table, so one was written into this
5039 // folder, whatever the walk below finds and wherever the
5040 // kept copy lives.
5041 .inspect(|table| note_opened(&mut opened_tables, &table_base_folder, table))
5042 // The digest was recomputed from THESE bytes, so recovery
5043 // can only prove the file is self-consistent — and it stops
5044 // at the trailer, meta and index, leaving every lazily-read
5045 // data block unexamined. Walk the blocks, exactly as the
5046 // retained copy's keep-decision does.
5047 .and_then(|table| {
5048 // A tight-space-punched copy carries the same
5049 // legitimate hole its retained twin does, and the walk
5050 // starts at the view's punch offset — which is zero on
5051 // an unrestricted open. Restrict it first, or its
5052 // reclaimed prefix reads as corruption and a healthy
5053 // duplicate lands in the corruption channel.
5054 let table = match trustworthy_restriction_bound(
5055 config,
5056 &*folder_fs,
5057 &table_path,
5058 table_id,
5059 &manifest_restriction,
5060 )? {
5061 Some(bound) => table.reopen_restricted(bound)?,
5062 None => table,
5063 };
5064 match block_verify_verdict(config, &folder_fs, &table_path, &table)? {
5065 BlockVerifyVerdict::Clean | BlockVerifyVerdict::DegradedButReadable => {
5066 Ok(())
5067 }
5068 BlockVerifyVerdict::Corrupt => Err(crate::Error::InvalidHeader(
5069 "block verification failed on this copy",
5070 )),
5071 // Nothing about the data was verified (the walk
5072 // could not size the parity trailers), so this copy
5073 // cannot be called healthy either.
5074 BlockVerifyVerdict::DegradedUnscanned => Err(
5075 crate::Error::InvalidHeader("this copy could not be verified"),
5076 ),
5077 }
5078 }),
5079 // Same split as the retained copy's read: a fault in the
5080 // ENVIRONMENT is not evidence about these bytes.
5081 Err(e) if is_environmental(&e) => return Err(e),
5082 Err(e) => Err(e),
5083 };
5084 // `block_verify_verdict` propagates an environmental failure
5085 // rather than grading it; that is the operator's to fix, not a
5086 // verdict on this copy.
5087 let verdict = match verdict {
5088 Err(e) if is_environmental(&e) => return Err(e),
5089 other => other,
5090 };
5091 match verdict {
5092 Ok(()) => {
5093 let reason = "duplicate table id; a complete copy is already held";
5094 excluded_files.push((table_path.clone(), reason.to_string()));
5095 discard_after_commit.push((
5096 Arc::clone(&folder_fs),
5097 table_path,
5098 reason.to_string(),
5099 ));
5100 }
5101 Err(e) => {
5102 let reason = format!(
5103 "damaged duplicate of table {table_id} (kept copy is intact): {e}"
5104 );
5105 // The corruption signal is real, but the coverage is
5106 // not lost: the complete copy of this id is retained.
5107 redundant_unreadable.insert(table_path.clone());
5108 set_aside_path(
5109 &folder_fs,
5110 &table_path,
5111 &reason,
5112 &mut unreadable_files,
5113 &mut discard_after_commit,
5114 );
5115 }
5116 }
5117 continue;
5118 }
5119
5120 // Hash the file and open it. A non-transient hashing failure (a bad
5121 // data sector) is FOLDED into the recover Result so the salvage arm
5122 // below can recover the table's intact blocks, instead of recording
5123 // the whole table unreadable — which the next open's orphan cleanup
5124 // would then delete. Table::recover would fail on the same bytes, so
5125 // skip it; block salvage opens with a placeholder digest and drops
5126 // only the unreadable blocks.
5127 // Whether THIS file is the one the committed manifest digested.
5128 // Two complete copies of an id can hold different generations, and
5129 // this is the only thing that tells them apart. A RESTRICTED
5130 // table's entry digests its live suffix, which a whole-file hash
5131 // never reproduces, so it stays `false` (uncomparable) rather than
5132 // wrongly reading as a mismatch.
5133 // The digest was taken above, before the duplicate verdict needed
5134 // it. An ENVIRONMENTAL hashing failure already propagated there; a
5135 // read that failed on the BYTES (a bad data sector, a corrupt
5136 // trailer) is folded into the recover Result so the
5137 // structural-failure salvage arm below recovers the intact blocks
5138 // (or records it unreadable with salvage off).
5139 let recovered = match own_digest {
5140 Ok(digest) => Table::recover(held_recover_params(
5141 config,
5142 table_path.clone(),
5143 digest,
5144 table_id,
5145 folder_fs.clone(),
5146 manifest_position,
5147 )),
5148 Err(e) => Err(e),
5149 };
5150
5151 // Remember what this table COVERS while its metadata is in hand. If
5152 // it ends up excluded, that coverage is what the rebuilt manifest
5153 // lost, and the report names it: older versions of those keys
5154 // survive elsewhere and become visible again, which no repair can
5155 // distinguish from "the key was simply never rewritten".
5156 if let Ok(t) = &recovered {
5157 note_opened(&mut opened_tables, &table_base_folder, t);
5158 let range = t.metadata.key_range.clone();
5159 // The bound is UNKNOWN when the table's sequence base lived in
5160 // the LOST manifest: that open passes offset 0, so
5161 // `get_highest_seqno` would report the on-disk LOCAL maximum
5162 // (normally 0 for a bulk-ingested SST) and an operator scoping
5163 // the affected history by it would stop far below the truth.
5164 // With a clean manifest record the offset was reused above,
5165 // so the bound is honest again.
5166 let seqno = (manifest_position.is_some()
5167 || !has_unrecoverable_ingest_offset(
5168 t.metadata.bulk_ingested,
5169 t.metadata.item_count,
5170 t.max_local_seqno(),
5171 ))
5172 .then(|| t.get_highest_seqno());
5173 coverage_by_path.insert(
5174 table_path.clone(),
5175 (range.min().clone(), range.max().clone(), seqno),
5176 );
5177 }
5178
5179 // Fail closed on a table whose bulk-ingest sequence offset cannot be
5180 // reconstructed. A bulk-ingested SST stores every entry at LOCAL seqno
5181 // 0 and relies on a manifest-only `global_seqno` for its effective MVCC
5182 // ordering; the on-disk seqnos carry no trace of it. The rebuilt
5183 // manifest hard-codes offset 0, so keeping such a table would make its
5184 // entries appear OLDER than they are — visible to snapshots that never
5185 // saw them and sorted into the wrong L0 order. Drop it instead of
5186 // silently corrupting MVCC (see `has_unrecoverable_ingest_offset`).
5187 // ONLY without a clean manifest record: with one, the offset was
5188 // recovered above and the table keeps its real sequence position.
5189 if manifest_position.is_none()
5190 && matches!(&recovered, Ok(t) if has_unrecoverable_ingest_offset(
5191 t.metadata.bulk_ingested,
5192 t.metadata.item_count,
5193 t.max_local_seqno(),
5194 ))
5195 {
5196 drop(recovered); // release the file handle
5197 set_aside_path(
5198 &folder_fs,
5199 &table_path,
5200 "bulk-ingest sequence offset cannot be reconstructed from the SST",
5201 &mut unreadable_files,
5202 &mut discard_after_commit,
5203 );
5204 continue;
5205 }
5206
5207 // Rebuild the restricted view of a tight-space-PUNCHED SST. Tight-space
5208 // compaction reclaims a table's consumed prefix data blocks in place
5209 // (hole-punched, reading back as zeros) and records the exact bound in a
5210 // `.restrict-bound` sidecar beside the SST. A rebuilt manifest must
5211 // re-apply that restriction, or later reads and compactions traverse the
5212 // punched blocks and fail. The bound comes from the sidecar (the SST
5213 // itself is never mutated, so its whole-file checksum stays valid);
5214 // written strictly post-commit, a valid sidecar is itself proof of a
5215 // committed restriction, so its bound is honored directly (see below).
5216 // Whether the recovered view's restriction came from PUNCH GEOMETRY
5217 // with resurrection off: that bound is the straddling block's END
5218 // key, which can discard the block's still-live suffix rows — a
5219 // deliberate, non-salvage loss the report must carry (scoped by the
5220 // source's coverage), or `wal_replay_scope` stays `TailOnly` while
5221 // live rows were removed. An exact sidecar bound and the
5222 // resurrection-mode bound (which keeps the whole readable region)
5223 // lose nothing.
5224 let mut geometry_lossy = false;
5225 let recovered = 'restrict: {
5226 let Ok(table) = recovered else {
5227 break 'restrict recovered;
5228 };
5229 // The exact bound, from the committed manifest or its sidecar
5230 // mirror. See `trustworthy_restriction_bound`.
5231 let exact_bound = match trustworthy_restriction_bound(
5232 config,
5233 &*folder_fs,
5234 &table_path,
5235 table_id,
5236 &manifest_restriction,
5237 ) {
5238 Ok(bound) => bound,
5239 Err(e) => break 'restrict Err(e),
5240 };
5241
5242 // An exact bound is honored DIRECTLY, without probing the
5243 // below-bound prefix: whether or not the punch has run, reopening at
5244 // the bound is correct. If the prefix is not yet punched (the crash
5245 // window between the durable commit and the punch, or a punch deferred
5246 // by a live reader), the committed output already covers the dropped
5247 // prefix, so honoring resurrects nothing; if it is punched, the
5248 // reopened view digests only the live suffix. Reading the dead prefix
5249 // to decide is not just unnecessary — a persistently unreadable sector
5250 // there would otherwise discard the exact bound and, with salvage off,
5251 // drop the whole table despite its intact live suffix.
5252 // `reopen_restricted` reads only from the punch offset up, so a
5253 // genuinely unreadable SUFFIX still surfaces its error there.
5254 if let Some(bound) = &exact_bound {
5255 break 'restrict table
5256 .reopen_restricted_with(bound.clone(), held_descriptors(config));
5257 }
5258
5259 // No trustworthy exact bound. An unpunched table never carried a
5260 // restriction, so it opens unrestricted. A punched table lost its
5261 // exact bound and falls to the punch geometry: with resurrection
5262 // on, restrict to the FIRST key of the first readable block past
5263 // the punched region, keeping the whole ambiguous readable region
5264 // (its consumed rows resurrected, as the flag contracts). With
5265 // resurrection OFF the geometry is trusted only when the zeroed
5266 // blocks form a CLEAN prefix — the pattern of a fully successful
5267 // punch — and the bound is that prefix's straddling block's END
5268 // key, never resurrecting a superseded key. An IRREGULAR pattern
5269 // (a readable block below a zeroed one) is positive evidence of
5270 // failed punches, after which no geometry bound can separate
5271 // intact-but-consumed blocks from live ones: the table is set
5272 // aside (see `DerivedRestriction::IrregularPunch`), matching the
5273 // recovery-failure arm's fail-closed guard for the same state. A
5274 // fully-punched SST with no live data is set aside too, losing
5275 // nothing the flag could have kept.
5276 // ONE pass over the data blocks answers both questions below —
5277 // whether the zeros are a reclaim, and where the live region
5278 // starts.
5279 let geometry = match table.punch_geometry() {
5280 Ok(geometry) => geometry,
5281 Err(e) => break 'restrict Err(e),
5282 };
5283 match geometry.verdict {
5284 crate::table::PunchProbe::Unpunched => break 'restrict Ok(table),
5285 // UNPROVEN zeros on a punch-capable mount: a lost-sidecar
5286 // reclaim and damage are indistinguishable, and publishing
5287 // unrestricted would resurrect the boundary block's rows
5288 // below the committed restriction. Without resurrection
5289 // the table is set aside; with it, the geometry paths
5290 // below (which read zeroed BLOCKS, not holes) restrict
5291 // past the ambiguous region, accepting the re-exposure
5292 // the flag contracts.
5293 crate::table::PunchProbe::Unproven if !allow_resurrection => {
5294 drop(table);
5295 set_aside_path(
5296 &folder_fs,
5297 &table_path,
5298 "zeroed data blocks whose allocation state the backend \
5299 cannot attribute (a lost-sidecar punch and damage are \
5300 indistinguishable); a resurrection repair restricts past \
5301 the zeroed region instead",
5302 &mut unreadable_files,
5303 &mut discard_after_commit,
5304 );
5305 continue 'dirent;
5306 }
5307 crate::table::PunchProbe::Punched | crate::table::PunchProbe::Unproven => {
5308 use crate::table::DerivedRestriction;
5309 let derived = if allow_resurrection {
5310 match table.greedy_restriction_bound(&geometry) {
5311 Ok(Some(bound)) => Ok(DerivedRestriction::Bound(bound)),
5312 Ok(None) => Ok(DerivedRestriction::NoLiveData),
5313 Err(e) => Err(e),
5314 }
5315 } else {
5316 Ok(Table::conservative_restriction(&geometry))
5317 };
5318 match derived {
5319 Ok(DerivedRestriction::Bound(bound)) => {
5320 geometry_lossy = !allow_resurrection;
5321 break 'restrict table
5322 .reopen_restricted_with(bound, held_descriptors(config));
5323 }
5324 Err(e) => break 'restrict Err(e),
5325 Ok(
5326 reason @ (DerivedRestriction::NoLiveData
5327 | DerivedRestriction::IrregularPunch),
5328 ) => {
5329 // The flag decides this table's fate WITHIN this
5330 // run and nowhere else. Resurrection would have
5331 // kept the readable region; without it there is
5332 // no bound that separates consumed rows from
5333 // live ones, so the table is dropped. Nothing is
5334 // stashed for a later run to reconsider: a run
5335 // that stashed would hand the next one a
5336 // different directory than the one it derived
5337 // from, and the flag would stop being an input
5338 // and start being a state machine.
5339 let reason = match reason {
5340 DerivedRestriction::NoLiveData => {
5341 "fully hole-punched SST with no live data"
5342 }
5343 _ => {
5344 "partially punched SST with punch failures and no \
5345 trustworthy bound; the consumed/live boundary is \
5346 unknowable (a resurrection repair keeps the readable \
5347 region instead)"
5348 }
5349 };
5350 drop(table);
5351 set_aside_path(
5352 &folder_fs,
5353 &table_path,
5354 reason,
5355 &mut unreadable_files,
5356 &mut discard_after_commit,
5357 );
5358 continue 'dirent;
5359 }
5360 }
5361 }
5362 }
5363 };
5364
5365 match recovered {
5366 // In salvage mode a table whose whole-file recovery succeeded can
5367 // still hold corrupt data blocks (recovery is lazy on the data
5368 // section). Block-verify it; if any block is corrupt, salvage it
5369 // rather than keep a table that errors on read. Encrypted and
5370 // unencrypted tables take the SAME encryption-aware out-of-band
5371 // walk (block headers and payload checksums are plaintext; the
5372 // provider only decodes the meta block) — the recovered `table`
5373 // merely supplies the id the encrypted meta's AAD binds. Without
5374 // the provider the walk could not decode an encrypted meta block
5375 // and would misreport every healthy encrypted table as corrupt,
5376 // rewriting it on every repair.
5377 Ok(table) if salvage => {
5378 match verify_keep_decision(
5379 config,
5380 &folder_fs,
5381 &table_path,
5382 &table,
5383 allow_resurrection,
5384 true,
5385 )? {
5386 RepairKeepDecision::Keep => {
5387 record_best(
5388 &mut recovered_by_id,
5389 &mut unreadable_files,
5390 &mut redundant_unreadable,
5391 &mut discard_after_commit,
5392 &mut swap_after_commit,
5393 table_id,
5394 table,
5395 if geometry_lossy {
5396 Fidelity::GeometryRestricted
5397 } else {
5398 Fidelity::Complete
5399 },
5400 &folder_fs,
5401 &table_path,
5402 matches_manifest,
5403 )?;
5404 }
5405 RepairKeepDecision::Drop(reason) => {
5406 drop(table);
5407 set_aside_path(
5408 &folder_fs,
5409 &table_path,
5410 reason,
5411 &mut unreadable_files,
5412 &mut discard_after_commit,
5413 );
5414 }
5415 RepairKeepDecision::Salvage => {
5416 // A tight-space RESTRICTED punched SST whose live suffix
5417 // is ALSO corrupt (a rare double failure) is block-
5418 // salvaged like any other, then RE-RESTRICTED to its
5419 // original bound: salvage recovers the readable blocks
5420 // into a fresh, unpunched SST (dropping the zeroed prefix
5421 // and the corrupt blocks), and reopening that restricted
5422 // to the recorded bound masks the straddling block's
5423 // sub-bound rows again, so nothing superseded is
5424 // resurrected. A sidecar re-records the bound so a later
5425 // manifest-loss repair honors it (the fresh file is
5426 // unpunched). With resurrection on, the whole readable
5427 // region is kept instead. The live suffix is never
5428 // thrown away.
5429 let restrict_bound = table.restrict_lower_bound().cloned();
5430 drop(table);
5431 // The replacement is built at `{id}.repair-tmp` and
5432 // swapped onto `{id}` only after the manifest that
5433 // adopts it is durable. The source is never displaced
5434 // first: a crash would then leave the only copy
5435 // somewhere the retry does not scan, and the table's
5436 // keys would silently vanish from the rebuilt
5437 // manifest. Reading an untouched source and writing a
5438 // name no scan adopts also makes the retry repeat
5439 // exactly the same deterministic salvage.
5440 let output_path = repair_tmp_path(&table_path);
5441 match try_salvage_table(
5442 config,
5443 &folder_fs,
5444 allow_resurrection,
5445 TableSalvage {
5446 source: &table_path,
5447 table_path: &output_path,
5448 table_id,
5449 // The bound (when any) comes from the
5450 // restricted view and is re-imposed below,
5451 // so a punched source is never ambiguous
5452 // on this arm.
5453 reject_punched_without_bound: false,
5454 blob_rewrite: None,
5455 owner_promotions: None,
5456 recovered_position: manifest_position,
5457 },
5458 ) {
5459 Ok(SalvageOutcome::Salvaged(salvaged)) => {
5460 // Re-impose the tight-space restriction on the
5461 // salvaged output (fail-closed unless resurrection
5462 // is on), the shared path both salvage arms use.
5463 let table = restrict_salvaged_output(
5464 &*folder_fs,
5465 config,
5466 &output_path,
5467 salvaged,
5468 restrict_bound.clone(),
5469 allow_resurrection,
5470 )?;
5471 keep_salvaged_replacement(
5472 &mut recovered_by_id,
5473 &mut unreadable_files,
5474 &mut redundant_unreadable,
5475 &mut discard_after_commit,
5476 &mut swap_after_commit,
5477 table_id,
5478 table,
5479 &folder_fs,
5480 &table_path,
5481 output_path,
5482 )?;
5483 }
5484 Ok(SalvageOutcome::Unusable | SalvageOutcome::PunchedBoundLost) => {
5485 let reason = "verify found corrupt blocks; nothing salvageable";
5486 set_aside_path(
5487 &folder_fs,
5488 &table_path,
5489 reason,
5490 &mut unreadable_files,
5491 &mut discard_after_commit,
5492 );
5493 }
5494 // A TRANSIENT I/O salvage failure is retryable, and
5495 // nothing has moved: the source is untouched, so the
5496 // retry re-derives the same salvage from it. A
5497 // STRUCTURAL failure is genuine unsalvageability.
5498 Err(salvage_err) if is_environmental(&salvage_err) => {
5499 return Err(salvage_err);
5500 }
5501 Err(salvage_err) => {
5502 let reason = format!(
5503 "verify found corrupt blocks; salvage failed \
5504 ({salvage_err})"
5505 );
5506 set_aside_path(
5507 &folder_fs,
5508 &table_path,
5509 &reason,
5510 &mut unreadable_files,
5511 &mut discard_after_commit,
5512 );
5513 }
5514 }
5515 }
5516 }
5517 }
5518 // Salvage OFF: block-verify all the same. Whole-file recovery is
5519 // lazy on the data section, and the manifest digest is freshly
5520 // computed over whatever bytes are there — blessing a table with
5521 // a corrupt data block would LAUNDER the damage (the report
5522 // counts it recovered and `verify_integrity` passes while reads
5523 // of the affected block fail). The salvage flag only decides what
5524 // happens to a damaged table: rewritten (on) or set aside (off,
5525 // here), with the report pointing at the salvage-enabled repair.
5526 Ok(table) => {
5527 match verify_keep_decision(
5528 config,
5529 &folder_fs,
5530 &table_path,
5531 &table,
5532 allow_resurrection,
5533 false,
5534 )? {
5535 RepairKeepDecision::Keep => {
5536 record_best(
5537 &mut recovered_by_id,
5538 &mut unreadable_files,
5539 &mut redundant_unreadable,
5540 &mut discard_after_commit,
5541 &mut swap_after_commit,
5542 table_id,
5543 table,
5544 if geometry_lossy {
5545 Fidelity::GeometryRestricted
5546 } else {
5547 Fidelity::Complete
5548 },
5549 &folder_fs,
5550 &table_path,
5551 matches_manifest,
5552 )?;
5553 }
5554 // `Salvage` is unreachable with the flag off, but a
5555 // defensive fallthrough beats a panic in a repair path.
5556 decision @ (RepairKeepDecision::Drop(_) | RepairKeepDecision::Salvage) => {
5557 let reason = match decision {
5558 RepairKeepDecision::Drop(reason) => reason,
5559 _ => {
5560 "verification found corrupt data blocks; run a \
5561 salvage-enabled repair to rewrite the readable blocks"
5562 }
5563 };
5564 drop(table);
5565 set_aside_path(
5566 &folder_fs,
5567 &table_path,
5568 reason,
5569 &mut unreadable_files,
5570 &mut discard_after_commit,
5571 );
5572 }
5573 }
5574 }
5575 Err(e) if salvage => {
5576 // A TRANSIENT recovery failure (Io) is retryable and must NOT
5577 // be routed through salvage: salvaging a healthy SST would,
5578 // for a range-tombstone table, fail deterministically with
5579 // FeatureUnsupported (a non-Io error recorded as
5580 // unsalvageable), committing a manifest without the table —
5581 // turning a one-shot read failure into permanent loss.
5582 // Propagate the I/O error so a retry re-recovers it,
5583 // mirroring the verification / salvage-error paths.
5584 if is_environmental(&e) {
5585 return Err(e);
5586 }
5587 // A tight-space-punched SST that fails whole-file recovery still
5588 // carries its restriction in the CLEAN manifest, or in the
5589 // `.restrict-bound` sidecar mirroring it — but recovery produced
5590 // no `Table` to read the bound from. Same question, same answer as
5591 // the verification-failure arm above; here the bound is re-imposed
5592 // on the salvaged replacement instead of reopening the source.
5593 let restrict_bound = trustworthy_restriction_bound(
5594 config,
5595 &*folder_fs,
5596 &table_path,
5597 table_id,
5598 &manifest_restriction,
5599 )?;
5600 // A PUNCHED source with no trustworthy bound (missing / corrupt
5601 // sidecar) cannot be salvaged into an UNRESTRICTED output:
5602 // recovery produced no `Table` to derive a geometry bound from,
5603 // and salvage drops the zeroed prefix but re-emits the straddling
5604 // block's sub-bound rows, which would resurrect with nothing to
5605 // restrict them. Fail closed: set it aside. An UNPUNCHED source
5606 // (the common corrupt-table case) has no zeroed prefix and
5607 // salvages normally. Resurrection-on skips the guard, accepting
5608 // the re-exposure.
5609 if restrict_bound.is_none()
5610 && !allow_resurrection
5611 && source_prefix_is_punched(&*folder_fs, &table_path)?
5612 {
5613 // The flag decides this within THIS run: with
5614 // resurrection on, the same source salvages and keeps
5615 // its readable region. Nothing is stashed for a later
5616 // run to reconsider.
5617 let reason = "punched SST with no recoverable restriction bound \
5618 (missing / corrupt sidecar and failed recovery); a \
5619 resurrection repair keeps its readable region instead";
5620 set_aside_path(
5621 &folder_fs,
5622 &table_path,
5623 reason,
5624 &mut unreadable_files,
5625 &mut discard_after_commit,
5626 );
5627 continue;
5628 }
5629 // Whole-file recovery failed structurally; try block-level
5630 // salvage. The recoverable blocks go into `{id}.repair-tmp`,
5631 // which is swapped onto `{id}` only once the manifest that
5632 // adopts it is durable — so a crash mid-repair leaves the
5633 // directory exactly as the retry expects to find it.
5634 let output_path = repair_tmp_path(&table_path);
5635 // Fail closed when the salvage walk reveals a punched source
5636 // with no recoverable bound: the pre-salvage first-bytes
5637 // probe above catches a punched FIRST block, but a partial
5638 // punch can leave that block intact while later prefix
5639 // blocks are zeroed — only the walk's dropped extents expose
5640 // those.
5641 let reject_punched = restrict_bound.is_none() && !allow_resurrection;
5642 match try_salvage_table(
5643 config,
5644 &folder_fs,
5645 allow_resurrection,
5646 TableSalvage {
5647 source: &table_path,
5648 table_path: &output_path,
5649 table_id,
5650 reject_punched_without_bound: reject_punched,
5651 blob_rewrite: None,
5652 owner_promotions: None,
5653 recovered_position: manifest_position,
5654 },
5655 ) {
5656 Ok(SalvageOutcome::Salvaged(salvaged)) => {
5657 // Re-impose the tight-space restriction on the salvaged
5658 // output (fail-closed unless resurrection is on), the
5659 // shared path both salvage arms use.
5660 let table = restrict_salvaged_output(
5661 &*folder_fs,
5662 config,
5663 &output_path,
5664 salvaged,
5665 restrict_bound,
5666 allow_resurrection,
5667 )?;
5668 keep_salvaged_replacement(
5669 &mut recovered_by_id,
5670 &mut unreadable_files,
5671 &mut redundant_unreadable,
5672 &mut discard_after_commit,
5673 &mut swap_after_commit,
5674 table_id,
5675 table,
5676 &folder_fs,
5677 &table_path,
5678 output_path,
5679 )?;
5680 }
5681 Ok(SalvageOutcome::Unusable) => {
5682 let reason = format!("unrecoverable ({e}); nothing salvageable");
5683 set_aside_path(
5684 &folder_fs,
5685 &table_path,
5686 &reason,
5687 &mut unreadable_files,
5688 &mut discard_after_commit,
5689 );
5690 }
5691 Ok(SalvageOutcome::PunchedBoundLost) => {
5692 // The flag decides this within THIS run: with
5693 // resurrection on the same source salvages and keeps
5694 // its readable region.
5695 let reason = format!(
5696 "punched SST with no recoverable restriction bound \
5697 (missing / corrupt sidecar and failed recovery, punched \
5698 extents found during salvage): {e}; a resurrection repair \
5699 keeps its readable region instead"
5700 );
5701 set_aside_path(
5702 &folder_fs,
5703 &table_path,
5704 &reason,
5705 &mut unreadable_files,
5706 &mut discard_after_commit,
5707 );
5708 }
5709 // Transient I/O salvage failure: nothing moved, so the
5710 // retry re-derives the same salvage from the untouched
5711 // source. A structural failure is recorded.
5712 Err(salvage_err) if is_environmental(&salvage_err) => {
5713 return Err(salvage_err);
5714 }
5715 Err(salvage_err) => {
5716 let reason =
5717 format!("recovery failed ({e}); salvage failed ({salvage_err})");
5718 set_aside_path(
5719 &folder_fs,
5720 &table_path,
5721 &reason,
5722 &mut unreadable_files,
5723 &mut discard_after_commit,
5724 );
5725 }
5726 }
5727 }
5728 Err(e) => {
5729 // A TRANSIENT recovery failure (Io) is retryable: recording it
5730 // unreadable commits a manifest without the still-in-place
5731 // file, which the next open's orphan cleanup then DELETES —
5732 // permanent loss from a one-shot read failure. Propagate it so
5733 // a retry re-recovers the table; only a structural failure is a
5734 // genuine unreadable report.
5735 if is_environmental(&e) {
5736 return Err(e);
5737 }
5738 // The rebuilt manifest omits this file, so it is removed once
5739 // that manifest is durable: a file both omitted and left in
5740 // place is an orphan the next open must sweep, and an open
5741 // that cannot sweep it fails.
5742 let reason = e.to_string();
5743 set_aside_path(
5744 &folder_fs,
5745 &table_path,
5746 &reason,
5747 &mut unreadable_files,
5748 &mut discard_after_commit,
5749 );
5750 }
5751 }
5752 }
5753 }
5754
5755 // A pooled file's id is above every table written into its folder: a
5756 // claim takes the lowest pooled id, and a write numbered above an id
5757 // passes over the pooled ones below it. A file being written when the
5758 // power went may sit below a table another write finished first; that
5759 // table lists it among the files its folder was still writing (see
5760 // `file_pool::Writing`). So an empty file is a table that lost its bytes
5761 // only when a table of its folder (one that opened as a table, kept, set
5762 // aside or left out as a duplicate, or a salvaged copy) is above it and
5763 // was finished after it was written: reported as a loss nothing can
5764 // scope. Any other is a pooled file or an output the power loss cut
5765 // short, never a table: it holds no rows and is no loss, and like every
5766 // file the rebuilt manifest does not name it is reported left out and
5767 // goes once that manifest is durable. A file with bytes that opens as no
5768 // table is not counted: nothing says a table was ever written there.
5769 //
5770 // A table emptied to nothing with no table of its folder finished after
5771 // it reads as pooled here: nothing on disk tells the two apart. A table
5772 // is synced before any edit names it, so only damage outside a crash
5773 // empties one; a durable mark on every pooled file would cost a data sync
5774 // per file, the sync the pool removes, and reporting every empty file as
5775 // a loss would call for a full replay on every repair of a pooled tree.
5776 for (table_id, folder, table_path, folder_fs) in empty_unnamed {
5777 let written = written_before_one_of(
5778 table_id,
5779 recovered_by_id
5780 .values()
5781 .filter(|c| c.path.parent() == Some(folder.as_path()))
5782 .map(|c| (c.table.id(), c.table.metadata.writing_below.as_slice()))
5783 .chain(
5784 opened_tables
5785 .get(&folder)
5786 .into_iter()
5787 .flatten()
5788 .map(|(id, below)| (*id, below.as_slice())),
5789 ),
5790 );
5791 if written {
5792 scanned_table_ids.insert(table_id);
5793 set_aside_path(
5794 &folder_fs,
5795 &table_path,
5796 "an empty table file below a table finished after it: its bytes were lost",
5797 &mut unreadable_files,
5798 &mut discard_after_commit,
5799 );
5800 } else {
5801 let reason = "an empty table file no table was written into";
5802 excluded_files.push((table_path.clone(), reason.to_string()));
5803 discard_after_commit.push((folder_fs, table_path, reason.to_string()));
5804 }
5805 }
5806
5807 Ok(TableScan {
5808 recovered_by_id,
5809 unreadable_files,
5810 redundant_unreadable,
5811 excluded_files,
5812 coverage_by_path,
5813 discard_after_commit,
5814 swap_after_commit,
5815 live_sidecars,
5816 scanned_table_ids,
5817 })
5818}
5819
5820/// Phase 2 of [`repair_tree`]: build a manifest from what the scan recovered,
5821/// commit it, and carry out exactly the removals and swaps that commit
5822/// authorizes.
5823///
5824/// The caller holds the directory lock for the whole of this.
5825///
5826/// # Errors
5827///
5828/// Propagates the commit failure. A failure AFTER the commit surfaces as
5829/// [`crate::Error::RepairedButUnopened`] carrying the finished report — the
5830/// repair is durable, so the obligation it names must not be lost.
5831#[cfg(feature = "std")]
5832fn rebuild_from_scan(
5833 config: &Config,
5834 allow_resurrection: bool,
5835 manifest_referenced: Option<CommittedManifest>,
5836 scan: TableScan,
5837 first_tree_id: crate::tree::inner::TreeId,
5838 outstanding: Option<RepairReport>,
5839 #[cfg(zstd_any)] damaged_dictionaries: &[crate::file::DictId],
5840) -> crate::Result<RepairReport> {
5841 let TableScan {
5842 recovered_by_id,
5843 mut unreadable_files,
5844 redundant_unreadable,
5845 mut excluded_files,
5846 coverage_by_path,
5847 mut discard_after_commit,
5848 mut swap_after_commit,
5849 live_sidecars,
5850 scanned_table_ids,
5851 } = scan;
5852
5853 // Collect the best copy per id, carrying each candidate's completeness so
5854 // `salvaged` can be derived from the tables that actually make the
5855 // manifest — after the blob-dependency filtering below, not before (a
5856 // salvaged table dropped for an unrecoverable blob dependency must not
5857 // count, or `salvaged` could exceed `recovered`). A lossy copy superseded
5858 // by a complete duplicate is likewise already gone from the candidates.
5859 // Each entry keeps its SOURCE path (the scanned `{id}` file, not a
5860 // salvage replacement's `{id}.repair-tmp`): it is the key into
5861 // `coverage_by_path` when a kept lossy copy's loss is scoped below.
5862 let mut recovered_tables: Vec<(Table, Fidelity, PathBuf, Arc<dyn crate::fs::Fs>)> =
5863 recovered_by_id
5864 .into_values()
5865 .map(|c| (c.table, c.fidelity, c.path, c.fs))
5866 .collect();
5867
5868 // Newest first, by DESCENDING recency key. For a flush table that key is
5869 // an id it took from the counter when it started (flushes are serialized,
5870 // so a higher one is later content); its own id is not, since a pooled
5871 // file's id was taken with its batch. A COMPACTION output's own id is NOT
5872 // that signal either: the id is taken when the compaction starts writing,
5873 // while newer flushes can install first, and an intra-L0 output is
5874 // appended at the BACK of L0 regardless of its id — so outputs persist
5875 // their newest INPUT's recency in meta (`recency`) and sort by it, with id
5876 // as the tie-break (equal recency means one descends from the other, and
5877 // an output's id is above every input's, so the higher id is the
5878 // superseding copy). A table's
5879 // highest seqno is NOT the signal either: callers may assign seqnos
5880 // explicitly, so an older table can top out above a newer one on an
5881 // unrelated key. The key is total, so repeating a repair over the same
5882 // files reproduces the same tree instead of inheriting the directory
5883 // scan's order.
5884 recovered_tables
5885 .sort_by_key(|(t, ..)| (std::cmp::Reverse(t.l0_recency()), std::cmp::Reverse(t.id())));
5886
5887 // KV-separated (blob) trees additionally carry a blob-file list. Discover the
5888 // blob files from the `blobs/` folder (no manifest to filter against) and
5889 // record them in the rebuilt manifest with the matching `TreeType::Blob` so
5890 // the tree reopens (the reopened tree's type must match its config's
5891 // `kv_separation_opts`). Fragmentation stats are NOT reconstructable from a
5892 // directory scan (they are derived from compaction history), so they start
5893 // empty: blob GC is advisory and re-learns them over time. The empty start
5894 // never drops live data; it only resets GC's view of reclaimable space.
5895 //
5896 // Runs BEFORE the L0 runs are built: a table whose indirections point into a
5897 // blob file this scan could NOT recover must not be published (see the
5898 // dependency check below), so the surviving blob ids have to be known first.
5899 let mut blob_rewrites: crate::HashMap<
5900 crate::vlog::BlobFileId,
5901 crate::salvage::BlobFileRewrite,
5902 > = crate::HashMap::default();
5903 // Fresh-id blob replacements this run publishes; removed on ANY exit —
5904 // an error, a cancellation — before the manifest commit disarms the
5905 // guard (see `PublishedBlobReplacements`).
5906 let mut published_blob_replacements = PublishedBlobReplacements::new(config, first_tree_id);
5907 let blob_files_salvaged: Vec<(PathBuf, String)> = Vec::new();
5908 let mut blob_frag = crate::blob_tree::FragmentationMap::default();
5909 // Damaged blob originals whose replacement is in the rebuilt manifest.
5910 // Set aside only after `persist_version` — see `BlobRecovery::stale`.
5911 let mut stale_blob_originals: Vec<(PathBuf, String)> = Vec::new();
5912 // Blob files the rebuilt manifest omits because nothing references them.
5913 // Removed only after `persist_version`, for the same reason.
5914 let mut unreferenced_blob_files: Vec<PathBuf> = Vec::new();
5915 let (tree_type, mut blob_file_list) = if config.kv_separation_opts.is_some() {
5916 if let Some(p) = &config.recovery_progress {
5917 p.set_phase(crate::RecoveryPhase::RecoveringBlobFiles);
5918 }
5919 // The candidate reference set, taken from the tables recovered so far
5920 // (a conservative superset of the post-filter truth): lets the blob
5921 // scan skip salvaging invalid files nothing can reach, and reserves
5922 // every referenced id in the fresh-id allocator. A table whose
5923 // reference section is structurally unreadable contributes nothing
5924 // here rather than aborting the whole repair — the dependency filter
5925 // below sets exactly that table aside (its dependencies are
5926 // unknowable), so nothing that survives can point at an id this set
5927 // missed. Environmental errors still propagate: a transient read
5928 // failure says nothing about the section.
5929 let mut referenced_blob_ids: crate::HashSet<crate::vlog::BlobFileId> =
5930 crate::HashSet::default();
5931 for (table, ..) in &recovered_tables {
5932 match table.list_blob_file_references() {
5933 Ok(links) => {
5934 for link in links.into_iter().flatten() {
5935 referenced_blob_ids.insert(link.blob_file_id);
5936 }
5937 }
5938 Err(e) if is_environmental(&e) => return Err(e),
5939 Err(e) => {
5940 log::warn!(
5941 "repair: table {} has an unreadable blob-reference section \
5942 ({e}); the dependency filter will set it aside",
5943 table.id(),
5944 );
5945 }
5946 }
5947 }
5948 let recovery = recover_blob_files(
5949 config,
5950 &mut published_blob_replacements,
5951 &referenced_blob_ids,
5952 manifest_referenced.as_ref().map(|m| &m.blob_frontiers),
5953 manifest_referenced.as_ref().map(|m| &m.blob_checksums),
5954 )?;
5955 unreadable_files.extend(recovery.unreadable);
5956 excluded_files.extend(recovery.excluded);
5957 blob_rewrites = recovery.rewrites;
5958 blob_frag = recovery.frag;
5959 stale_blob_originals = recovery.stale;
5960 // Foreign names, duplicates and unreadable blob files: same post-commit
5961 // removal as the superseded originals (see `BlobRecovery::discard`).
5962 discard_after_commit.extend(
5963 recovery
5964 .discard
5965 .into_iter()
5966 .map(|(path, note)| (Arc::clone(&config.fs), path, note)),
5967 );
5968 let map: crate::HashMap<crate::vlog::BlobFileId, crate::vlog::BlobFile> =
5969 recovery.files.into_iter().map(|bf| (bf.id(), bf)).collect();
5970 (TreeType::Blob, BlobFileList::new(map))
5971 } else {
5972 // A Standard rebuild must not swallow a blob tree: with the manifest
5973 // gone the open's tree-type check never ran, so a configuration that
5974 // omits kv-separation would otherwise publish a manifest that opens
5975 // fine while reads return encoded indirection handles as user values
5976 // and the blob files stay orphaned. The recovered tables' own
5977 // `linked_blob_files` sections prove the store's type; fail closed
5978 // with the same mismatch error a healthy open raises. A table whose
5979 // section cannot be READ proves nothing either way — publishing it
5980 // would be permission granted by ignorance, so it is set aside like
5981 // the kv path's dependency filter does (environmental errors
5982 // propagate for a retry).
5983 let candidates = core::mem::take(&mut recovered_tables);
5984 recovered_tables.reserve(candidates.len());
5985 for candidate in candidates {
5986 let is_blob_backed = match candidate.0.list_blob_file_references() {
5987 Ok(refs) => refs.is_some_and(|r| !r.is_empty()),
5988 Err(e) if is_environmental(&e) => return Err(e),
5989 Err(e) => {
5990 let (table, ..) = candidate;
5991 set_aside_table(
5992 table,
5993 &format!(
5994 "blob-file reference list unreadable ({e}) on a standard \
5995 rebuild; the table cannot prove its tree type"
5996 ),
5997 &mut unreadable_files,
5998 &mut discard_after_commit,
5999 );
6000 continue;
6001 }
6002 };
6003 if is_blob_backed {
6004 log::error!(
6005 "repair: table {} references blob files, so this store is a \
6006 KV-separated (blob) tree; rebuilding a Standard manifest \
6007 over it would return indirection handles as values — \
6008 configure kv separation and retry",
6009 candidate.0.id(),
6010 );
6011 return Err(crate::Error::TreeTypeMismatch {
6012 requested: TreeType::Standard,
6013 actual: TreeType::Blob,
6014 });
6015 }
6016 recovered_tables.push(candidate);
6017 }
6018 (
6019 TreeType::Standard,
6020 BlobFileList::new(crate::HashMap::default()),
6021 )
6022 };
6023
6024 // Drop any recovered table that still references a blob file the scan could
6025 // not recover: publishing the pair yields a manifest that opens fine while
6026 // a read of an affected key resolves a handle into a blob file that is not
6027 // there. A table whose `linked_blob_files` section cannot be read is treated
6028 // the same way (its dependencies are unknown, so it cannot be proven safe).
6029 // A table referencing a RESHAPED blob file — one the blob scan salvaged
6030 // into a compacted copy, or recovered with a punched frontier — is instead
6031 // REWRITTEN through the salvage pipeline: its handles are re-targeted at
6032 // the relocated records and only entries whose record no longer exists are
6033 // dropped, so intact live data is never discarded over a reshaped
6034 // dependency.
6035 // Shared with the ownership pass below, which rewrites tables again.
6036 let blob_rewrites = Arc::new(blob_rewrites);
6037 if config.kv_separation_opts.is_some() {
6038 // Frontiers of the punched-but-intact blob files (the `DropBelow`
6039 // rewrite entries): a handle below one dereferences zeroed bytes.
6040 // Empty on the common path, so no table's handles are scanned.
6041 let punched_frontiers: crate::HashMap<crate::vlog::BlobFileId, u64> = blob_rewrites
6042 .iter()
6043 .filter_map(|(id, rw)| match rw {
6044 crate::salvage::BlobFileRewrite::DropBelow(f) => Some((*id, *f)),
6045 crate::salvage::BlobFileRewrite::Remap { .. } => None,
6046 })
6047 .collect();
6048 let mut admitted: Vec<AdmittedTable> = Vec::with_capacity(recovered_tables.len());
6049 for (table, fidelity, source_path, source_fs) in recovered_tables {
6050 // One reference read drives everything below: the missing-id check
6051 // and the rewrite decision.
6052 let links = match table.list_blob_file_references() {
6053 Ok(links) => links,
6054 Err(e) if is_environmental(&e) => return Err(e),
6055 Err(e) => {
6056 set_aside_table(
6057 table,
6058 &format!("blob-file reference list unreadable ({e})"),
6059 &mut unreadable_files,
6060 &mut discard_after_commit,
6061 );
6062 continue;
6063 }
6064 };
6065 // A referenced id is unrecoverable only when the scan neither kept
6066 // it NOR salvaged it into a replacement: a `Remap` entry means the
6067 // records live on under a fresh id, and the rewrite below
6068 // retargets this table's handles at it.
6069 if let Some(l) = links.as_ref().and_then(|links| {
6070 links.iter().find(|l| {
6071 !blob_file_list.contains_key(l.blob_file_id)
6072 && !blob_rewrites.contains_key(&l.blob_file_id)
6073 })
6074 }) {
6075 set_aside_table(
6076 table,
6077 &format!("blob file {} is not recoverable", l.blob_file_id),
6078 &mut unreadable_files,
6079 &mut discard_after_commit,
6080 );
6081 continue;
6082 }
6083 // Whether this table's handles must be rewritten: any reference to
6084 // a SALVAGED blob file (its records moved to a fresh id), or a
6085 // handle that actually lies below a punched blob's frontier (a
6086 // pre-relocation SST file left behind by a crash — the id-presence
6087 // check cannot see it).
6088 let mut needs_rewrite = false;
6089 if let Some(links) = &links {
6090 if links.iter().any(|l| {
6091 matches!(
6092 blob_rewrites.get(&l.blob_file_id),
6093 Some(crate::salvage::BlobFileRewrite::Remap { .. })
6094 )
6095 }) {
6096 needs_rewrite = true;
6097 } else if !punched_frontiers.is_empty()
6098 && links
6099 .iter()
6100 .any(|l| punched_frontiers.contains_key(&l.blob_file_id))
6101 {
6102 match handle_below_blob_frontier(&table, &punched_frontiers) {
6103 Ok(hit) => needs_rewrite = hit.is_some(),
6104 Err(e) if is_environmental(&e) => return Err(e),
6105 Err(e) => {
6106 set_aside_table(
6107 table,
6108 &format!("blob handles unreadable ({e})"),
6109 &mut unreadable_files,
6110 &mut discard_after_commit,
6111 );
6112 continue;
6113 }
6114 }
6115 }
6116 }
6117 admitted.push(((table, fidelity, source_path, source_fs), needs_rewrite));
6118 }
6119 // Derived from the rows themselves, every time: a table file that is
6120 // simply gone takes its owners with it and leaves no trace the scan
6121 // could gate this on.
6122 // A row the walk cannot read leaves it unknown whether an object it
6123 // borrows has an owner elsewhere, and two owners would charge the
6124 // object twice: no ownership is handed then.
6125 let promotions = match owner_promotions(&admitted, &blob_rewrites) {
6126 Ok(promotions) => promotions,
6127 Err(e) if is_environmental(&e) => return Err(e),
6128 Err(e) => {
6129 log::warn!("repair: no blob object ownership handed on: a table's rows ({e})");
6130 admitted.iter().map(|_| None).collect()
6131 }
6132 };
6133 let mut kept: Vec<(Table, Fidelity, PathBuf, Arc<dyn crate::fs::Fs>)> =
6134 Vec::with_capacity(admitted.len());
6135 for (((table, fidelity, source_path, source_fs), needs_rewrite), promote) in
6136 admitted.into_iter().zip(promotions)
6137 {
6138 if !needs_rewrite && promote.is_none() {
6139 kept.push((table, fidelity, source_path, source_fs));
6140 continue;
6141 }
6142 // Rewrite through the salvage pipeline: re-emit every entry with
6143 // re-targeted handles, dropping only entries whose blob record no
6144 // longer exists. The rewritten table counts as salvaged (its
6145 // content may be lossy relative to the original).
6146 //
6147 // The copy is built at `{id}.repair-tmp` — a name no scan adopts —
6148 // and swapped onto `{id}` only AFTER the manifest commit. Displacing
6149 // the source first would open a window in which a crash leaves no
6150 // readable copy where the retry looks, and publishing the copy under
6151 // a name the scan DOES adopt would leave a crash with both readable,
6152 // so the retry would rebuild one history into L0 twice. Nothing is
6153 // carried across runs: the rewrite reads an untouched source, and a
6154 // leftover temp is garbage the retry replaces.
6155 let source_id = table.id();
6156 let path = (*table.path).clone();
6157 let output_path = repair_tmp_path(&path);
6158 // A RESTRICTED source's bound must be re-imposed on the copy: the
6159 // rewrite re-emits the straddling block's sub-bound rows, which the
6160 // restriction hides, so an unrestricted copy would resurrect them.
6161 let restrict_bound = table.restrict_lower_bound().cloned();
6162 // The source already carries its recovered ingest offset and L0
6163 // recency (from a clean manifest record, or the scan's own
6164 // admission); the rewrite preserves local seqnos and content, so
6165 // the copy reopens at the same position. ZERO is a VALID allocated
6166 // offset — the first ingestion on a fresh counter commits offset
6167 // 0 — not a sentinel for absence, so it is forwarded
6168 // unconditionally: a table that reached this point was already
6169 // admitted by the scan, and re-entering the fail-closed offset
6170 // exclusion here would reject (and delete) its healthy replacement.
6171 let source_position = ManifestPosition::of_table(&table);
6172 let fs = table.fs.clone();
6173 // A table rewritten only to hand ownership loses nothing: it stays
6174 // as it was, its owners unchanged, if the rewrite cannot be made.
6175 let kept_as_is = if needs_rewrite {
6176 // Its descriptor too: a replacement may be renamed over below,
6177 // which a backend refuses for an open file (Windows).
6178 release_held(config, table);
6179 None
6180 } else {
6181 Some(table)
6182 };
6183 let fidelity = if needs_rewrite {
6184 Fidelity::Salvaged
6185 } else {
6186 fidelity
6187 };
6188 match try_salvage_table(
6189 config,
6190 &fs,
6191 allow_resurrection,
6192 TableSalvage {
6193 source: &path,
6194 table_path: &output_path,
6195 table_id: source_id,
6196 reject_punched_without_bound: false,
6197 blob_rewrite: Some(Arc::clone(&blob_rewrites)),
6198 owner_promotions: promote.map(Arc::new),
6199 recovered_position: Some(source_position),
6200 },
6201 ) {
6202 Ok(SalvageOutcome::Salvaged(rewritten)) => {
6203 if let Some(table) = kept_as_is {
6204 release_held(config, table);
6205 }
6206 let rewritten = restrict_salvaged_output(
6207 &*fs,
6208 config,
6209 &output_path,
6210 rewritten,
6211 restrict_bound,
6212 allow_resurrection,
6213 )?;
6214 let restricted = rewritten.restrict_lower_bound().is_some();
6215 if let Some(pending) = swap_after_commit
6216 .iter_mut()
6217 .find(|(_, tmp, ..)| *tmp == path)
6218 {
6219 // A source that is itself a replacement this repair
6220 // built is superseded by the copy, which takes the
6221 // replacement's name before the manifest can name it:
6222 // a crash between the commit and the swap then leaves
6223 // the copy the manifest names where the next open
6224 // finishes swaps from, not under a name no scan reads.
6225 // The replacement is no longer needed: a failed commit
6226 // is retried from the untouched original.
6227 let checksum = rewritten.checksum();
6228 let bound = rewritten.restrict_lower_bound().cloned();
6229 release_held(config, rewritten);
6230 commit_repair_tmp(&*fs, &output_path, &path, config.sync_mode, restricted)?;
6231 let published = Table::recover(held_recover_params(
6232 config,
6233 path.clone(),
6234 checksum,
6235 source_id,
6236 Arc::clone(&fs),
6237 Some(source_position),
6238 ))?;
6239 let published = match bound {
6240 Some(bound) => {
6241 published.reopen_restricted_with(bound, held_descriptors(config))?
6242 }
6243 None => published,
6244 };
6245 pending.3 = restricted;
6246 kept.push((published, fidelity, source_path, source_fs));
6247 } else {
6248 swap_after_commit.push((Arc::clone(&fs), output_path, path, restricted));
6249 kept.push((rewritten, fidelity, source_path, source_fs));
6250 }
6251 }
6252 // A retryable failure leaves the source where it was found, so
6253 // the retry re-derives the same rewrite from it; nothing to
6254 // restore.
6255 Err(e) if is_environmental(&e) => return Err(e),
6256 Ok(_) | Err(_) if kept_as_is.is_some() => {
6257 if let Some(table) = kept_as_is {
6258 log::warn!(
6259 "repair: table {source_id} keeps the objects it borrows unowned: \
6260 its rewrite produced nothing usable"
6261 );
6262 discard_unreferenced(&*fs, &output_path, config.sync_mode)?;
6263 kept.push((table, fidelity, source_path, source_fs));
6264 }
6265 }
6266 Ok(SalvageOutcome::Unusable | SalvageOutcome::PunchedBoundLost) => {
6267 set_aside_path(
6268 &fs,
6269 &path,
6270 "blob-handle rewrite produced nothing",
6271 &mut unreadable_files,
6272 &mut discard_after_commit,
6273 );
6274 }
6275 Err(e) => {
6276 set_aside_path(
6277 &fs,
6278 &path,
6279 &format!("blob-handle rewrite failed ({e})"),
6280 &mut unreadable_files,
6281 &mut discard_after_commit,
6282 );
6283 }
6284 }
6285 }
6286 recovered_tables = kept;
6287 }
6288
6289 // A crashed compaction that FINALIZED its outputs before its version edit
6290 // committed leaves BOTH histories on disk — and so does a committed
6291 // compaction whose input deletion never finished. The recorded lineage
6292 // settles it: an output whose inputs ALL survived into this rebuild is
6293 // DERIVED — the inputs are the complete history, a future compaction
6294 // re-folds them — and publishing both would apply the same merge
6295 // operands twice on every read. Exclusion cascades correctly in a single
6296 // pass against the PRE-exclusion id set: an intermediate output excluded
6297 // for its surviving inputs may itself be named in a later output's
6298 // lineage, but every exclusion keeps the excluded table's own inputs, so
6299 // the remaining set always bottoms out at lineage-less leaves carrying
6300 // the full history. An output with PARTIALLY surviving inputs is KEPT —
6301 // it is the only complete copy of its span — alongside the surviving
6302 // inputs, and each overlap is reported below as ambiguity coverage so a
6303 // reconciling deployment knows reads in that span may double-apply
6304 // operands. Presence is judged AFTER the blob-dependency filter above: an
6305 // input dropped for an unrecoverable blob does not count as surviving —
6306 // and only a COMPLETE recovery counts at all: a salvaged or
6307 // geometry-restricted input already lost records, so its id proves
6308 // nothing about the output's contents surviving elsewhere, and trading a
6309 // healthy output for damaged inputs would convert recoverable data into
6310 // permanent loss.
6311 let present_ids: crate::HashSet<TableId> = recovered_tables
6312 .iter()
6313 .filter(|(_, fidelity, ..)| fidelity.is_complete())
6314 .map(|(t, ..)| t.id())
6315 .collect();
6316 // Per-candidate geometry and ancestry, captured over EVERY candidate
6317 // before any exclusion: the union pass and the transitive-supersession
6318 // closure below must reason about tables the first pass already dropped
6319 // (an excluded intermediate output still stands between a retained
6320 // descendant and its ultimate inputs).
6321 let all_ranges: crate::HashMap<TableId, (UserKey, UserKey, SeqNo)> = recovered_tables
6322 .iter()
6323 .map(|(t, ..)| {
6324 (
6325 t.id(),
6326 (
6327 t.metadata.key_range.min().clone(),
6328 t.metadata.key_range.max().clone(),
6329 t.get_highest_seqno(),
6330 ),
6331 )
6332 })
6333 .collect();
6334 let lineage_by_id: crate::HashMap<TableId, Vec<TableId>> = recovered_tables
6335 .iter()
6336 .filter_map(|(t, ..)| t.metadata.lineage.clone().map(|l| (t.id(), l)))
6337 .collect();
6338 // The final `bool` records whether the overlapping OUTPUT was
6339 // transformed: its filter may have removed keys inside the overlap, so
6340 // the kept input's copies are not byte-identical duplicates there.
6341 let mut lineage_partial: Vec<(TableId, PathBuf, UserKey, UserKey, Option<SeqNo>, bool)> =
6342 Vec::new();
6343 let mut inputs_superseded: crate::HashSet<TableId> = crate::HashSet::default();
6344 // Files excluded as REDUNDANT (a derived output whose inputs all
6345 // survived, an input a complete output fully covers): their content
6346 // lives on in the kept tables, so they join `excluded_files` with NO
6347 // `lost_coverage` (nothing was lost). The id set feeds the ancestry
6348 // walk below.
6349 let mut redundant_excluded_ids: crate::HashSet<TableId> = crate::HashSet::default();
6350 {
6351 let candidates = core::mem::take(&mut recovered_tables);
6352 for candidate in candidates {
6353 let lineage = candidate.0.metadata.lineage.clone();
6354 match lineage.as_deref() {
6355 // A TRANSFORMED output never trades back for its inputs: a
6356 // compaction filter removed records from its window, so the
6357 // resurrected inputs would revive them. It takes the partial
6358 // arm below instead and SUPERSEDES the inputs it covers.
6359 Some(inputs)
6360 if !candidate.0.metadata.lineage_transformed
6361 && !inputs.is_empty()
6362 && inputs
6363 .iter()
6364 .all(|id| *id != candidate.0.id() && present_ids.contains(id)) =>
6365 {
6366 let (table, _, path, fs) = candidate;
6367 let table_id = table.id();
6368 log::info!(
6369 "repair: table {table_id} is a compaction output whose inputs \
6370 {inputs:?} all survived; excluding the derived copy so its \
6371 merge operands are not applied twice",
6372 );
6373 drop(table);
6374 let reason = "derived output of an uncommitted compaction whose inputs \
6375 all survived; excluded so its merge operands are not \
6376 applied twice";
6377 redundant_excluded_ids.insert(table_id);
6378 excluded_files.push((path.clone(), reason.to_string()));
6379 discard_after_commit.push((fs, path, reason.to_string()));
6380 }
6381 Some(inputs) => {
6382 // Partial survival: the output is the only complete copy
6383 // of the LOST inputs' span, so it must be kept — and each
6384 // surviving input the output's key range fully COVERS is
6385 // superseded by it (the compaction read the whole input,
6386 // so every one of its records is in a range-covering
6387 // output) and is excluded, or reads in the overlap apply
6388 // the same merge operands twice; `lost_coverage` cannot
6389 // fix that, since the documented reconciliation would see
6390 // the duplicated operand as surviving and subtract its
6391 // WAL record instead of removing the folded copy. Only a
6392 // COMPLETE output supersedes. An input the output covers
6393 // partially (its remainder belonged to a LOST sibling
6394 // output of the same run) is kept — dropping it would
6395 // lose live records — and the residual overlap is
6396 // reported.
6397 let cmp = config.comparator.as_ref();
6398 for input in inputs {
6399 if *input == candidate.0.id() {
6400 continue;
6401 }
6402 let Some((in_min, in_max, in_hi)) = all_ranges.get(input) else {
6403 continue;
6404 };
6405 let out_range = &candidate.0.metadata.key_range;
6406 if cmp.compare(out_range.min(), in_max) == core::cmp::Ordering::Greater
6407 || cmp.compare(in_min, out_range.max()) == core::cmp::Ordering::Greater
6408 {
6409 continue;
6410 }
6411 let covers = candidate.1.is_complete()
6412 && cmp.compare(out_range.min(), in_min) != core::cmp::Ordering::Greater
6413 && cmp.compare(in_max, out_range.max()) != core::cmp::Ordering::Greater;
6414 if covers {
6415 inputs_superseded.insert(*input);
6416 continue;
6417 }
6418 let lo =
6419 if cmp.compare(out_range.min(), in_min) == core::cmp::Ordering::Less {
6420 in_min
6421 } else {
6422 out_range.min()
6423 };
6424 let hi =
6425 if cmp.compare(out_range.max(), in_max) == core::cmp::Ordering::Less {
6426 out_range.max()
6427 } else {
6428 in_max
6429 };
6430 lineage_partial.push((
6431 *input,
6432 candidate.2.clone(),
6433 lo.clone(),
6434 hi.clone(),
6435 Some(candidate.0.get_highest_seqno().min(*in_hi)),
6436 candidate.0.metadata.lineage_transformed,
6437 ));
6438 }
6439 recovered_tables.push(candidate);
6440 }
6441 _ => recovered_tables.push(candidate),
6442 }
6443 }
6444 }
6445 // Sibling-UNION supersession: the outputs of one rotated run record the
6446 // same lineage plus an adjacency link (`lineage_prev`), so an UNBROKEN
6447 // surviving chain of COMPLETE outputs proves its combined range has no
6448 // gap a lost sibling could hide — an input inside that union is fully
6449 // redundant even when no single output contains it. A broken chain
6450 // (a predecessor that did not survive) proves nothing and unions
6451 // nothing. A chain that spans the run END TO END — from the FIRST
6452 // output (no `lineage_prev`) to the one carrying the `lineage_last`
6453 // marker — is the run's COMPLETE output set and supersedes every input
6454 // outright: keys outside the written ranges were consumed by the merge
6455 // (obsolete versions, annihilated weak deletes, filter removals), not
6456 // lost. That is what preserves a committed TRANSFORMING compaction —
6457 // its outputs no longer contain the filtered records, so only full-run
6458 // supersession retires the inputs that still do.
6459 {
6460 let mut by_id: crate::HashMap<TableId, usize> = crate::HashMap::default();
6461 for (idx, (t, fidelity, ..)) in recovered_tables.iter().enumerate() {
6462 if fidelity.is_complete() && t.metadata.lineage.is_some() {
6463 by_id.insert(t.id(), idx);
6464 }
6465 }
6466 let same_lineage = |a: usize, b: usize| {
6467 recovered_tables.get(a).map(|(t, ..)| &t.metadata.lineage)
6468 == recovered_tables.get(b).map(|(t, ..)| &t.metadata.lineage)
6469 };
6470 let mut next_of: crate::HashMap<TableId, TableId> = crate::HashMap::default();
6471 for &idx in by_id.values() {
6472 let Some((t, ..)) = recovered_tables.get(idx) else {
6473 continue;
6474 };
6475 if let Some(prev) = t.metadata.lineage_prev
6476 && by_id.get(&prev).is_some_and(|&p| same_lineage(p, idx))
6477 {
6478 next_of.insert(prev, t.id());
6479 }
6480 }
6481 let cmp = config.comparator.as_ref();
6482 for (&head_id, &head_idx) in &by_id {
6483 let Some((head, ..)) = recovered_tables.get(head_idx) else {
6484 continue;
6485 };
6486 // A chain HEAD: its predecessor is absent from the surviving set
6487 // (or it is the run's first output).
6488 if head
6489 .metadata
6490 .lineage_prev
6491 .is_some_and(|prev| by_id.contains_key(&prev))
6492 {
6493 continue;
6494 }
6495 // Walk the unbroken chain; ranges follow the run's key order, so
6496 // the union is [head.min, tail.max]. A chain that runs from the
6497 // run's FIRST output (no `lineage_prev`) to its LAST (the
6498 // `lineage_last` marker, written only by a writer that owned the
6499 // WHOLE run — never by a parallel or tight-space slice, whose
6500 // first output also has no persisted predecessor) is the
6501 // COMPLETE output set: it supersedes every input outright, no
6502 // range check needed. Requiring BOTH ends keeps a surviving
6503 // slice from claiming the whole run.
6504 let open_lo = head.metadata.lineage_prev.is_none();
6505 let union_min = head.metadata.key_range.min().clone();
6506 let mut union_max = head.metadata.key_range.max().clone();
6507 let mut open_hi = head.metadata.lineage_last;
6508 let mut cursor = head_id;
6509 while let Some(&next) = next_of.get(&cursor) {
6510 if let Some(&next_idx) = by_id.get(&next)
6511 && let Some((t, ..)) = recovered_tables.get(next_idx)
6512 {
6513 union_max = t.metadata.key_range.max().clone();
6514 open_hi = t.metadata.lineage_last;
6515 }
6516 cursor = next;
6517 }
6518 let complete_run = open_lo && open_hi;
6519 let Some(inputs) = &head.metadata.lineage else {
6520 continue;
6521 };
6522 for input in inputs {
6523 if by_id.contains_key(input) {
6524 continue;
6525 }
6526 // `all_ranges`, not the surviving set: superseding an id the
6527 // first pass already excluded is what lets the ancestry
6528 // closure below reach ITS inputs.
6529 let Some((in_min, in_max, _)) = all_ranges.get(input) else {
6530 continue;
6531 };
6532 if complete_run
6533 || (cmp.compare(&union_min, in_min) != core::cmp::Ordering::Greater
6534 && cmp.compare(in_max, &union_max) != core::cmp::Ordering::Greater)
6535 {
6536 inputs_superseded.insert(*input);
6537 }
6538 }
6539 }
6540 }
6541
6542 // Transitive-supersession closure over the lineage edges: a superseded
6543 // table's content is PROVEN incorporated into a retained output, and it
6544 // in turn incorporated its own inputs — so those are redundant too,
6545 // even when the retaining output's lineage never names them (it names
6546 // the intermediate). Records an intermediate fold dropped (obsolete
6547 // versions, annihilated weak-delete pairs) return shadowed or
6548 // re-annihilating, and filter removals stay removed with their carrier,
6549 // so following the edge loses nothing live. Follows SUPERSEDED ids
6550 // only, never exclusions: a derived output excluded in the first pass
6551 // was dropped because its inputs are the preferred history — walking
6552 // through it would discard the very tables that exclusion kept.
6553 {
6554 let mut worklist: Vec<TableId> = inputs_superseded.iter().copied().collect();
6555 while let Some(id) = worklist.pop() {
6556 let Some(inputs) = lineage_by_id.get(&id) else {
6557 continue;
6558 };
6559 for input in inputs {
6560 if all_ranges.contains_key(input) && inputs_superseded.insert(*input) {
6561 worklist.push(*input);
6562 }
6563 }
6564 }
6565 }
6566
6567 // Second pass: drop the inputs a surviving COMPLETE output fully covers.
6568 // Their residual-coverage entries (recorded against a DIFFERENT,
6569 // partially-covering output before the covering one was seen) go with
6570 // them — the ambiguity those entries reported no longer exists.
6571 if !inputs_superseded.is_empty() {
6572 let candidates = core::mem::take(&mut recovered_tables);
6573 for candidate in candidates {
6574 if inputs_superseded.contains(&candidate.0.id()) {
6575 let (table, _, path, fs) = candidate;
6576 log::info!(
6577 "repair: table {} is fully covered by a surviving compaction \
6578 output; excluding it so its merge operands are not applied twice",
6579 table.id(),
6580 );
6581 drop(table);
6582 let reason = "input fully covered by a surviving compaction output; \
6583 excluded so its merge operands are not applied twice";
6584 excluded_files.push((path.clone(), reason.to_string()));
6585 discard_after_commit.push((fs, path, reason.to_string()));
6586 } else {
6587 recovered_tables.push(candidate);
6588 }
6589 }
6590 lineage_partial.retain(|(input, ..)| !inputs_superseded.contains(input));
6591 }
6592
6593 // A RESIDUAL overlap that survived every supersession pass fails closed
6594 // in two configurations. Against a TRANSFORMED output — always: the
6595 // kept input may hold a key the filter removed inside the overlap,
6596 // reads would resurrect it, and `lost_coverage` cannot repair the
6597 // deletion because a compaction-filter verdict is not an external-WAL
6598 // event. Under a merge operator — for any residual: the kept input's
6599 // records inside the overlap are also folded into the kept output, and
6600 // publishing both applies the same operands twice on every read — a
6601 // multiplicity the reported replay cannot remove, the same rule the
6602 // legacy ambiguity below takes. This is the crash shape a parallel
6603 // compaction leaves when input cleanup is partial (an input spanning
6604 // several sub-compaction ranges survives while no single output or
6605 // provable chain covers it whole), and equally a serial run's
6606 // lost-sibling window. Value-only residuals against UNTRANSFORMED
6607 // outputs proceed to the report: their duplicate records are
6608 // byte-identical and reads dedupe them.
6609 if let Some((input, output_path, ..)) = lineage_partial
6610 .iter()
6611 .find(|(.., transformed)| config.merge_operator.is_some() || *transformed)
6612 {
6613 log::error!(
6614 "repair: input table {} overlaps the surviving compaction output {} \
6615 that folded part of it, and no surviving output set covers the \
6616 input whole; publishing both would resurrect filter-removed keys \
6617 or double-apply merge operands, and no replay can undo either",
6618 input,
6619 output_path.display(),
6620 );
6621 return Err(crate::Error::Unrecoverable);
6622 }
6623
6624 // Ancestry audit for every RETAINED transformed output: each id its
6625 // lineage names must be either superseded (its content provably
6626 // incorporated) or itself an excluded derived output whose OWN inputs
6627 // pass the same test — the exclusion moved the history onto them. A
6628 // live, unsuperseded carrier still holds rows the transform removed,
6629 // no range test can see that (the transform may have emptied the
6630 // overlap entirely), and no replay can re-delete them: fail closed.
6631 for (table, ..) in &recovered_tables {
6632 if !table.metadata.lineage_transformed {
6633 continue;
6634 }
6635 let Some(inputs) = &table.metadata.lineage else {
6636 continue;
6637 };
6638 let mut visited: crate::HashSet<TableId> = crate::HashSet::default();
6639 let mut worklist: Vec<TableId> = inputs.clone();
6640 while let Some(id) = worklist.pop() {
6641 if id == table.id() || !visited.insert(id) || inputs_superseded.contains(&id) {
6642 continue;
6643 }
6644 if redundant_excluded_ids.contains(&id) {
6645 // The excluded intermediate's history lives on in its own
6646 // inputs; audit those instead.
6647 if let Some(carried) = lineage_by_id.get(&id) {
6648 worklist.extend(carried.iter().copied());
6649 }
6650 continue;
6651 }
6652 if all_ranges.contains_key(&id) {
6653 log::error!(
6654 "repair: table {} carries history the transformed compaction \
6655 output {} filtered, and no surviving output set proves that \
6656 history incorporated; publishing both would resurrect the \
6657 filter-removed rows, and no replay can re-delete them",
6658 id,
6659 table.id(),
6660 );
6661 return Err(crate::Error::Unrecoverable);
6662 }
6663 }
6664 }
6665
6666 // Only blob files a surviving table REFERENCES go into the manifest.
6667 // An unreferenced one holds no reachable value by definition, and
6668 // admitting it would strand it there forever: repair cannot rebuild
6669 // fragmentation stats from a directory scan, and blob GC retires a
6670 // file only once its recorded stale bytes reach the totals it never
6671 // gets. That also settles what a crashed earlier attempt leaves
6672 // behind — a fully written salvage replacement that the crash kept
6673 // out of any manifest — which would otherwise be admitted as an
6674 // ordinary blob and pin a whole copy per failed attempt. Judged against
6675 // the FINAL table set — after the lineage dedup above — so a blob
6676 // referenced only by an excluded table (a derived output, a superseded
6677 // input) is not pinned by a reference the rebuilt manifest no longer
6678 // holds.
6679 if config.kv_separation_opts.is_some() {
6680 let mut referenced: crate::HashSet<crate::vlog::BlobFileId> = crate::HashSet::default();
6681 for (table, ..) in &recovered_tables {
6682 for link in table.list_blob_file_references()?.into_iter().flatten() {
6683 referenced.insert(link.blob_file_id);
6684 }
6685 }
6686 // Carry each dropped file's RECOVERED path, never a path rebuilt from
6687 // its id: the scan accepts a noncanonical spelling (`blobs/01` for id
6688 // 1), and removing the reconstructed `blobs/1` would answer `NotFound`
6689 // — reporting success while leaving the real file behind for the next
6690 // open to sweep, which then fails if that removal is refused.
6691 let dropped: Vec<(crate::vlog::BlobFileId, PathBuf)> = blob_file_list
6692 .iter()
6693 .filter(|bf| !referenced.contains(&bf.id()))
6694 .map(|bf| (bf.id(), bf.path().to_path_buf()))
6695 .collect();
6696 for (id, path) in dropped {
6697 log::debug!(
6698 "blob file {id} is referenced by no recovered table; leaving it out \
6699 of the rebuilt manifest and removing it"
6700 );
6701 blob_file_list.remove(id);
6702 blob_frag.remove(&id);
6703 // Removed POST-COMMIT (see the sweep below): until the manifest is
6704 // durable the file is still what a failed attempt's inputs point
6705 // at, and a retry re-derives the same decision from a fresh scan.
6706 unreferenced_blob_files.push(path);
6707 }
6708 }
6709
6710 // Live sidecars are judged here, against the FINAL rebuilt set: one whose
6711 // table survived is preserved (the open and the next scrub read it), an
6712 // ORPHAN — its table excluded, superseded, or never recovered — is
6713 // removed post-commit. The open sweeps such orphans unconditionally and
6714 // fails on a refused removal, so leaving one would let a successful
6715 // repair be followed by an open failure on the very same file.
6716 {
6717 let final_ids: crate::HashSet<TableId> =
6718 recovered_tables.iter().map(|(t, ..)| t.id()).collect();
6719 for (id, path, fs) in live_sidecars {
6720 if final_ids.contains(&id) {
6721 continue;
6722 }
6723 discard_after_commit.push((
6724 fs,
6725 path,
6726 "orphaned sidecar; its table did not survive the rebuild".to_string(),
6727 ));
6728 }
6729 }
6730
6731 // `salvaged` is a subset of `recovered`, so derive it from the tables that
6732 // survived every filter above. The live progress counter follows the same
6733 // rule: a candidate displaced by deduplication or dropped by dependency
6734 // filtering never counts, so the snapshot cannot claim more tables than
6735 // the rebuilt manifest holds.
6736 // Only block-salvage REWRITES: a geometry-restricted original is lossy
6737 // (it contributes coverage below) but was never salvaged, and it arises
6738 // in plain repairs where `salvaged` is documented to stay zero.
6739 let salvaged = recovered_tables
6740 .iter()
6741 .filter(|(_, fidelity, ..)| *fidelity == Fidelity::Salvaged)
6742 .count();
6743 // A KEPT lossy copy — a salvaged rewrite that dropped corrupt blocks (or
6744 // blob records), or a geometry-restricted original whose derived bound
6745 // may exclude the straddling block's live suffix — names nothing in
6746 // `unreadable_files`, so without an entry here `wal_replay_scope()` would
6747 // answer `TailOnly` while older persisted changes were in fact lost. The
6748 // loss is scoped by the
6749 // SOURCE's coverage captured during the scan — NOT by the replacement's
6750 // own metadata, which only bounds what SURVIVED: salvage may have
6751 // dropped the block holding the source's outermost keys or its highest
6752 // seqno, and a range/ceiling derived from the survivors would exclude
6753 // exactly the lost part. A source whose metadata never parsed
6754 // (whole-file recovery failed before the coverage was captured) is
6755 // unscopable and joins `unknowable_losses` instead.
6756 let mut salvaged_unknowable: Vec<PathBuf> = Vec::new();
6757 let salvaged_coverage: Vec<(PathBuf, UserKey, UserKey, Option<SeqNo>)> = recovered_tables
6758 .iter()
6759 .filter(|(_, fidelity, ..)| !fidelity.is_complete())
6760 .filter_map(|(_, _, source_path, _)| {
6761 if let Some((lo, hi, bound)) = coverage_by_path.get(source_path) {
6762 Some((source_path.clone(), lo.clone(), hi.clone(), *bound))
6763 } else {
6764 salvaged_unknowable.push(source_path.clone());
6765 None
6766 }
6767 })
6768 .collect();
6769 // Every table carries its L0 recency key (a table without one is refused
6770 // as it is read), so the order rebuilt from it is the order the live tree
6771 // kept and ties in it need no report.
6772 let recovered_tables: Vec<Table> = recovered_tables.into_iter().map(|(t, ..)| t).collect();
6773 publish_repaired_manifest(
6774 config,
6775 RepairPublication {
6776 recovered_tables,
6777 tree_type,
6778 blob_file_list,
6779 blob_frag,
6780 published_blob_replacements,
6781 unreadable_files,
6782 redundant_unreadable,
6783 excluded_files,
6784 lost_coverage_scoped: (salvaged_coverage, lineage_partial, salvaged_unknowable),
6785 coverage_by_path,
6786 manifest_referenced,
6787 scanned_table_ids,
6788 discard_after_commit,
6789 swap_after_commit,
6790 stale_blob_originals,
6791 unreferenced_blob_files,
6792 blob_files_salvaged,
6793 salvaged,
6794 first_tree_id,
6795 outstanding,
6796 #[cfg(zstd_any)]
6797 damaged_dictionaries,
6798 },
6799 )
6800}
6801
6802/// Everything [`publish_repaired_manifest`] needs: the tables that made it
6803/// through, and every list whose contents the commit either authorizes to be
6804/// carried out (removals, swaps) or must be reported.
6805#[cfg(feature = "std")]
6806struct RepairPublication<'a> {
6807 recovered_tables: Vec<Table>,
6808 tree_type: TreeType,
6809 blob_file_list: BlobFileList,
6810 blob_frag: crate::blob_tree::FragmentationMap,
6811 published_blob_replacements: PublishedBlobReplacements<'a>,
6812 unreadable_files: Vec<(PathBuf, String)>,
6813 /// Damaged files whose id is covered by a retained complete copy: reported
6814 /// as corruption, but contributing no lost coverage.
6815 redundant_unreadable: crate::HashSet<PathBuf>,
6816 excluded_files: Vec<(PathBuf, String)>,
6817 #[expect(clippy::type_complexity, reason = "the three coverage channels")]
6818 lost_coverage_scoped: (
6819 Vec<(PathBuf, UserKey, UserKey, Option<SeqNo>)>,
6820 Vec<(TableId, PathBuf, UserKey, UserKey, Option<SeqNo>, bool)>,
6821 Vec<PathBuf>,
6822 ),
6823 coverage_by_path: crate::HashMap<PathBuf, (UserKey, UserKey, Option<SeqNo>)>,
6824 manifest_referenced: Option<CommittedManifest>,
6825 scanned_table_ids: crate::HashSet<TableId>,
6826 discard_after_commit: Vec<(Arc<dyn crate::fs::Fs>, PathBuf, String)>,
6827 swap_after_commit: Vec<(Arc<dyn crate::fs::Fs>, PathBuf, PathBuf, bool)>,
6828 stale_blob_originals: Vec<(PathBuf, String)>,
6829 unreferenced_blob_files: Vec<PathBuf>,
6830 blob_files_salvaged: Vec<(PathBuf, String)>,
6831 salvaged: usize,
6832 /// The first tree id the repair handed out: the descriptors cached under
6833 /// it and every later id are closed before the post-commit swaps.
6834 first_tree_id: crate::tree::inner::TreeId,
6835 /// The report of an earlier repair over the installed snapshot that never
6836 /// reached the caller: its losses are owed as well.
6837 outstanding: Option<RepairReport>,
6838 /// Dictionaries found damaged at the start; those still damaged when the
6839 /// report is assembled are set aside after the commit.
6840 #[cfg(zstd_any)]
6841 damaged_dictionaries: &'a [crate::file::DictId],
6842}
6843
6844/// Phase 3 of [`repair_tree`]: commit the rebuilt manifest, then carry out
6845/// exactly what that commit authorizes and assemble the report.
6846///
6847/// The commit is the pivot: before it the directory is untouched and any
6848/// failure leaves the retry the same bytes; after it the manifest is durable,
6849/// so a failure carries the finished report out through
6850/// [`crate::Error::RepairedButUnopened`] rather than discarding it.
6851///
6852/// # Errors
6853///
6854/// Propagates the commit failure, or wraps a post-commit one with the report.
6855#[cfg(feature = "std")]
6856fn publish_repaired_manifest(
6857 config: &Config,
6858 publication: RepairPublication<'_>,
6859) -> crate::Result<RepairReport> {
6860 let RepairPublication {
6861 recovered_tables,
6862 tree_type,
6863 blob_file_list,
6864 blob_frag,
6865 mut published_blob_replacements,
6866 unreadable_files,
6867 redundant_unreadable,
6868 excluded_files,
6869 lost_coverage_scoped: (salvaged_coverage, lineage_partial, salvaged_unknowable),
6870 coverage_by_path,
6871 manifest_referenced,
6872 scanned_table_ids,
6873 discard_after_commit,
6874 swap_after_commit,
6875 stale_blob_originals,
6876 unreferenced_blob_files,
6877 mut blob_files_salvaged,
6878 salvaged,
6879 first_tree_id,
6880 outstanding,
6881 #[cfg(zstd_any)]
6882 damaged_dictionaries,
6883 } = publication;
6884 // A rebuilt manifest whose highest table id is the LAST one cannot be
6885 // committed: the next open seeds its id allocator with `highest + 1`,
6886 // which overflows — a panic in checked builds, a wrap to an existing low
6887 // id in release builds (a later flush then collides with that file).
6888 // Mirrors the version-id and blob-id exhaustion guards: fail the repair
6889 // instead of publishing a tree that cannot allocate.
6890 if recovered_tables.iter().map(Table::id).max() == Some(crate::TableId::MAX) {
6891 log::error!(
6892 "repair: the table id space is exhausted (id {} is in use); a rebuilt \
6893 manifest would make the next open's id allocator overflow",
6894 crate::TableId::MAX,
6895 );
6896 return Err(crate::Error::Unrecoverable);
6897 }
6898 // The blob-file id space counts on the same rule. A HEALTHY,
6899 // still-referenced blob file at the last id needs no salvage, so the
6900 // fresh-id allocator's exhaustion (`next_blob_id == None`) is never
6901 // consulted for it — yet the next open seeds its blob allocator with
6902 // `max + 1` all the same, with the same overflow.
6903 if blob_file_list.iter().map(crate::vlog::BlobFile::id).max()
6904 == Some(crate::vlog::BlobFileId::MAX)
6905 {
6906 log::error!(
6907 "repair: the blob file id space is exhausted (id {} is referenced); a \
6908 rebuilt manifest would make the next open's blob id allocator overflow",
6909 crate::vlog::BlobFileId::MAX,
6910 );
6911 return Err(crate::Error::Unrecoverable);
6912 }
6913 if let Some(p) = &config.recovery_progress {
6914 p.tables_recovered_add(recovered_tables.len() as u64);
6915 // Blob files count on the same rule and for the same reason: the
6916 // reference filter above removes (and deletes) any the surviving tables
6917 // do not point at, so counting one at recovery time would claim a
6918 // recovery the rebuilt manifest does not hold.
6919 p.blob_files_recovered_add(blob_file_list.len() as u64);
6920 }
6921
6922 // Each recovered table becomes its own single-table L0 run. L0 permits
6923 // overlapping runs, so this is always legal regardless of key overlap;
6924 // background compaction collapses them into sorted lower levels later.
6925 // `Run::new` only returns `None` for an empty run, which `vec![t]` never is,
6926 // so no table is dropped here — but build the runs explicitly and derive the
6927 // recovered count from what actually lands in the manifest, so the report
6928 // can never overcount relative to the persisted version.
6929 let l0_runs = recovered_tables
6930 .iter()
6931 .cloned()
6932 .filter_map(|t| Run::new(vec![t]).map(Arc::new))
6933 .collect::<Vec<_>>();
6934 let recovered = l0_runs.len();
6935
6936 let mut levels = Vec::with_capacity(config.level_count.into());
6937 levels.push(Level::from_runs(l0_runs));
6938 for _ in 1..config.level_count {
6939 levels.push(Level::empty());
6940 }
6941
6942 // Next version id after the highest existing one: recovery takes the
6943 // highest snapshot, so the rebuilt one must be above every other. Only a
6944 // folder already holding `v{u64::MAX}` has no such id. A rebuilt snapshot
6945 // high in the range needs no headroom for later edits: the next one comes
6946 // from an open, which renumbers any tree in the top half to version 0.
6947 let version_id = match highest_existing_version_id(&*config.fs, &config.path)? {
6948 Some(max) => max.checked_add(1).ok_or_else(|| {
6949 log::error!(
6950 "repair: v{max} exists, so no snapshot id is left above it for the \
6951 rebuilt manifest",
6952 );
6953 crate::Error::Unrecoverable
6954 })?,
6955 None => 0,
6956 };
6957
6958 // Seeded with the punched prefixes' garbage: those frames can never be
6959 // observed by a future compaction, so an empty map would pin every punched
6960 // file's stale count below its whole-file metadata totals forever.
6961 //
6962 // The retention floor comes from the caller (`Config::repair_retention_floor`).
6963 // The lost manifest was the only record of which snapshots a past GC
6964 // compaction or `clear` invalidated, and the tables cannot stand in for
6965 // it: a GC compaction zeroes the seqnos of the rows it settles, so a
6966 // table's highest seqno reads as `0` exactly on the trees where history
6967 // WAS collected. Nor may the floor be guessed high (say, at the highest
6968 // persisted seqno): the external-WAL reconciliation that follows a repair
6969 // restores intermediate snapshots and reads them back, so a guessed floor
6970 // would refuse history the caller has just made whole. Only the
6971 // deployment that ran the compactions knows the watermark; it supplies
6972 // it, and the default (`0`) serves everything, as before the floor
6973 // existed.
6974 let version = Version::from_levels(version_id, tree_type, levels, blob_file_list, blob_frag)
6975 .with_retention_floor(config.repair_retention_floor);
6976
6977 // Register the dictionaries the recovered files name. A rebuilt version
6978 // starts with none, and nothing later puts them back once the write policy
6979 // stops naming them: a checkpoint copies exactly this list, so the snapshot
6980 // would carry the tables and not the dictionaries that decode them.
6981 //
6982 // Derived from the files themselves and intersected with what `dicts/`
6983 // actually holds, so a repair over a tree whose dictionary is genuinely
6984 // gone records no id it cannot honour.
6985 #[cfg(zstd_any)]
6986 let version = {
6987 let held = config.current_zstd_dictionaries();
6988 let ids: Vec<_> = version
6989 .referenced_dicts()
6990 .into_iter()
6991 .filter(|id| held.get(*id).is_some())
6992 .collect();
6993
6994 // STORE what the manifest is about to name. The set here holds the
6995 // caller-supplied dictionary as well as the folder's, so a repair of a
6996 // tree written before dictionaries were stored (tables naming an id,
6997 // no `dicts/` at all) would otherwise commit a manifest referencing
6998 // bytes that exist only in the caller's memory — and the reopen the
6999 // repair exists to enable would fail on the missing file. Writing is
7000 // idempotent by id, so a dictionary already on disk costs an `exists`.
7001 let folder = config.path.join(crate::file::DICTS_FOLDER);
7002 for id in &ids {
7003 if let Some(dict) = held.get(*id) {
7004 crate::dicts::write(
7005 &*config.fs,
7006 &folder,
7007 dict,
7008 config.encryption.as_deref(),
7009 config.sync_mode,
7010 )?;
7011 }
7012 }
7013
7014 if ids.is_empty() {
7015 version
7016 } else {
7017 version.with_dicts(ids)
7018 }
7019 };
7020
7021 // The LAST cancellation boundary: per-file checks only run before a file
7022 // starts, so a cancel requested during the final file's verification or
7023 // salvage would otherwise be silently outrun by the commit. From here on
7024 // the run is COMMITTING and then cleaning up, and cancellation is no
7025 // longer consulted (see `check_cancel`). An abort here — like every
7026 // earlier pre-commit exit — unwinds the fresh-id blob replacements
7027 // through the `published_blob_replacements` guard.
7028 check_cancel(config)?;
7029 if let Some(p) = &config.recovery_progress {
7030 p.set_phase(crate::RecoveryPhase::Committing);
7031 }
7032
7033 let mut warnings = vec![WARN_ALL_AT_L0, WARN_UNLOGGED_EDITS];
7034 if config.kv_separation_opts.is_some() {
7035 warnings.push(WARN_BLOB_FRAGMENTATION);
7036 }
7037
7038 // Join the exclusions against the coverage captured during the scan, plus
7039 // an entry per KEPT lossy salvage (see `salvaged_coverage`). An excluded
7040 // TABLE whose metadata never parsed has unknowable coverage — recorded
7041 // separately so `wal_replay_scope()` can force the full-history
7042 // obligation instead of silently answering as if nothing was lost. Only a
7043 // NUMERIC table candidate qualifies: a FOREIGN name (`notes.txt`) held no
7044 // table data by definition, so its exclusion lost nothing — reporting it
7045 // unknowable would demand an unbounded WAL archive over scribbles. Blob
7046 // files stay out for the same reason in the other direction: losing blob
7047 // content surfaces through the referencing tables (a lossy handle
7048 // rewrite, or their exclusion), which ARE covered above. Provenance, not
7049 // a path prefix, decides which is which: a level route may nest its
7050 // tables anywhere (even under the primary tree's blobs directory), so a
7051 // candidate is a table iff its parent is one of the scanned table
7052 // folders.
7053 let table_folders: Vec<PathBuf> = config
7054 .all_tables_folders()
7055 .into_iter()
7056 .map(|(folder, _)| folder)
7057 .collect();
7058 let mut lost_coverage: Vec<(PathBuf, UserKey, UserKey, Option<SeqNo>)> = Vec::new();
7059 let mut unknowable_losses: Vec<PathBuf> = Vec::new();
7060 let is_table_candidate = |path: &std::path::Path| {
7061 path.parent()
7062 .is_some_and(|parent| table_folders.iter().any(|f| f.as_path() == parent))
7063 && path.file_name().and_then(|n| n.to_str()).is_some_and(|n| {
7064 matches!(
7065 crate::file::TableDirEntry::classify(n),
7066 crate::file::TableDirEntry::Table(_)
7067 )
7068 })
7069 };
7070 // Redundant exclusions live in `excluded_files` and never reach this
7071 // loop: their content lives on in the kept tables, so they contribute
7072 // no coverage. A DAMAGED duplicate does reach it — the corruption signal
7073 // belongs in `unreadable_files` — but its coverage is not lost either, and
7074 // an unscopable entry here would send an external WAL over the whole
7075 // keyspace for a tree that lost nothing.
7076 for (path, _) in &unreadable_files {
7077 if redundant_unreadable.contains(path) {
7078 continue;
7079 }
7080 match coverage_by_path.get(path) {
7081 Some((lo, hi, seqno)) => {
7082 lost_coverage.push((path.clone(), lo.clone(), hi.clone(), *seqno));
7083 }
7084 None if is_table_candidate(path) => unknowable_losses.push(path.clone()),
7085 None => {}
7086 }
7087 }
7088 lost_coverage.extend(salvaged_coverage);
7089 lost_coverage.extend(
7090 lineage_partial
7091 .into_iter()
7092 .map(|(_, path, lo, hi, bound, _)| (path, lo, hi, bound)),
7093 );
7094 unknowable_losses.extend(salvaged_unknowable);
7095 // A table the CLEAN manifest referenced that the scan never even SAW: the
7096 // file is gone, so there is no directory entry to report through — and
7097 // the manifest records no key range for it, so the loss is unscopable.
7098 if let Some(referenced) = manifest_referenced {
7099 let primary_tables = config.path.join("tables");
7100 let mut missing: Vec<TableId> = referenced
7101 .tables
7102 .into_keys()
7103 .filter(|id| !scanned_table_ids.contains(id))
7104 .collect();
7105 missing.sort_unstable();
7106 for id in missing {
7107 log::warn!(
7108 "repair: table {id} is referenced by the recovered manifest but has \
7109 no file on disk; its loss is unscopable",
7110 );
7111 unknowable_losses.push(primary_tables.join(id.to_string()));
7112 }
7113 }
7114
7115 // The replacements are DURABLE the moment the manifest commits, so the
7116 // report names every one of them regardless of what the cleanup does: it
7117 // is what rides out inside `RepairedButUnopened`, the operator's only
7118 // account of which blobs were replaced. A removal after the commit only
7119 // annotates its entry.
7120 // The snapshot's copy is written before the cleanup and may be handed out
7121 // after it, so it says what holds either way; this call's report then
7122 // says what the cleanup did.
7123 let salvage_report_base = blob_files_salvaged.len();
7124 blob_files_salvaged.extend(stale_blob_originals.iter().map(|(path, note)| {
7125 (
7126 path.clone(),
7127 format!("{note}; original removed once the repair finishes"),
7128 )
7129 }));
7130
7131 // Every damaged dictionary still under its name is set aside once the
7132 // commit is durable (see `set_aside_damaged_dictionaries`). The report's
7133 // copy below is what an open hands out when the run ends before the
7134 // report reaches the caller, and by then the files may already be aside,
7135 // so the copy names them now. A dictionary the store above rewrote from a
7136 // supplied copy reads intact again and stays.
7137 #[cfg(zstd_any)]
7138 let damaged_dictionaries = {
7139 let folder = config.path.join(crate::file::DICTS_FOLDER);
7140 let mut entries = Vec::new();
7141 for &id in damaged_dictionaries {
7142 if crate::dicts::is_still_damaged(
7143 &*config.fs,
7144 &folder,
7145 id,
7146 config.encryption.as_deref(),
7147 )? {
7148 entries.push((
7149 crate::dicts::damaged_path(&folder, id),
7150 format!(
7151 "dictionary {id} failed its integrity check; moved aside once the \
7152 repair finishes"
7153 ),
7154 ));
7155 }
7156 }
7157 entries
7158 };
7159 #[cfg(not(zstd_any))]
7160 let damaged_dictionaries = Vec::new();
7161
7162 let mut report = RepairReport {
7163 recovered,
7164 salvaged,
7165 unreadable: unreadable_files.len(),
7166 unreadable_files,
7167 excluded_files,
7168 lost_coverage,
7169 unknowable_losses,
7170 blob_files_salvaged,
7171 damaged_dictionaries,
7172 method: REPAIR_METHOD,
7173 warnings,
7174 id: 0,
7175 includes: Vec::new(),
7176 };
7177 // Appended after this run's entries, so `salvage_report_base` still
7178 // indexes them.
7179 if let Some(earlier) = outstanding {
7180 fold_report(&mut report, earlier);
7181 }
7182 report.id = report_marker::id_for(&report, version_id, config.clock.as_ref());
7183
7184 // The report exists only in this call, and the commit below may end with
7185 // its outcome unknown: a power loss can still keep the rebuilt snapshot,
7186 // and the next open then finds nothing to repair. The snapshot holds the
7187 // report, so it is there exactly when the snapshot is installed, and an
7188 // open hands it out from there.
7189 let version = version.with_repair_report(Some(Arc::from(report_marker::encode(&report))));
7190
7191 // Persist with the tree's own runtime config, not defaults: it drives the
7192 // manifest framing (checksum algorithm, page ECC, footer mirror, manifest
7193 // KV checksums), so defaulting it would rewrite a recovered tree's manifest
7194 // metadata to settings it never used. The last live runtime config died with
7195 // the lost manifest; the config supplied to `repair` is the authoritative
7196 // replacement.
7197 let persisted = crate::version::persist_version(
7198 &config.path,
7199 &version,
7200 config.comparator.name(),
7201 &*config.fs,
7202 Arc::new(config.initial_runtime_config.clone()),
7203 config.encryption.clone(),
7204 config.sync_mode,
7205 );
7206 // The guard stays ARMED until the commit is decided: every fallible
7207 // `persist_version` step before it installs the snapshot (manifest
7208 // create / encode / finish, the log's creation, and the rename) leaves the
7209 // replacements unreferenced garbage a retry would re-derive; keeping
7210 // them would stack another fresh-id copy per failed attempt, walking
7211 // recovery toward ENOSPC under exactly the tight-space conditions it
7212 // targets. The ONE fallible step after the install is the directory sync,
7213 // so on failure the name on disk decides which world this is: the rebuilt
7214 // snapshot under its name means the manifest is or may become published and the
7215 // replacements are the files it references. A probe that cannot tell
7216 // proves nothing, and fail-safe is to PRESERVE: the worst a preserved
7217 // orphan costs is disk space a later successful repair's reference
7218 // filtering reclaims, while a wrong deletion breaks a published manifest
7219 // permanently.
7220 // A failure after a PROVEN install must still CARRY the report: the
7221 // rebuilt snapshot is installed on disk, so the retry opens it without a
7222 // repair and answers with no report at all — losing the replay
7223 // obligation exactly like a cleanup failure would. Recording the error
7224 // routes it through the same report-carrying exit. A proven NOT-INSTALLED
7225 // failure returns bare (nothing was published, the guard stays armed,
7226 // the retry's own repair produces a fresh report). An INCONCLUSIVE probe
7227 // also returns bare: without proof of the install the report may
7228 // describe a repair that never committed, and a consumer replaying its
7229 // obligation against the still-live OLD generation could duplicate merge
7230 // operands — while the blob replacements are preserved (the probe's own
7231 // fail-safe: their worst cost is disk space) and no post-commit cleanup
7232 // runs, so nothing of a possibly-live generation is touched. The report
7233 // rides in the rebuilt snapshot on every bare exit: an open hands it out
7234 // exactly when that snapshot is the installed one.
7235 let mut post_commit_error: Option<crate::Error> = None;
7236 match persisted {
7237 Ok(()) => published_blob_replacements.disarm(),
7238 Err(e) => match probe_install(&*config.fs, &config.path, version_id) {
7239 // When the persist renamed the snapshot into place and could not
7240 // make its withdrawal durable, a power loss may bring the name
7241 // back: its absence proves nothing then.
7242 InstallProbe::NotInstalled if !matches!(e, crate::Error::InstallOutcomeUnknown(_)) => {
7243 return Err(e);
7244 }
7245 // A visible name is not yet a durable install: the persist's own
7246 // directory sync failed, so a power loss can still take the name
7247 // back. Only a sync that succeeds now makes the rebuilt snapshot
7248 // the installed generation; until then the repair has not
7249 // committed, and answering as committed would hand out a replay
7250 // scope for a generation that may not survive.
7251 InstallProbe::Installed => {
7252 published_blob_replacements.disarm();
7253 if let Err(sync) = config
7254 .fs
7255 .sync_directory_with(&config.path, config.sync_mode)
7256 {
7257 log::warn!(
7258 "repair: the rebuilt snapshot v{version_id} is visible but its \
7259 install cannot be made durable ({sync}); its report rides in it",
7260 );
7261 return Err(match e {
7262 crate::Error::InstallOutcomeUnknown(_) => e,
7263 other => crate::Error::InstallOutcomeUnknown(Box::new(other)),
7264 });
7265 }
7266 post_commit_error = Some(e);
7267 }
7268 InstallProbe::NotInstalled | InstallProbe::Inconclusive => {
7269 published_blob_replacements.disarm();
7270 return Err(e);
7271 }
7272 },
7273 }
7274
7275 // The manifest is DURABLE from this point on (or the probe could not
7276 // prove otherwise): the repair happened, and its report must survive
7277 // every later failure. A bare error from the cleanup below would
7278 // discard the only report — once the filesystem fault clears, the next
7279 // open sweeps the leftover itself and `open_or_repair` answers with no
7280 // report at all, hiding the committed repair's lost coverage from an
7281 // external-WAL consumer. The first post-commit failure is therefore
7282 // RECORDED, the remaining cleanup is skipped, and the completed report
7283 // rides out inside [`Error::RepairedButUnopened`].
7284
7285 // A rebuilt snapshot is a complete generation on its own. Sweep every stale
7286 // edit log so nothing is replayed on top of it: the lost manifest's
7287 // generation left its log under an OLDER snapshot id (the rebuilt snapshot
7288 // uses `max(v*) + 1`), so removing only `edits-{version_id}` would normally
7289 // miss it. Drop every `edits-*` but the fresh snapshot's own, which the
7290 // persist created empty so its first append syncs no directory. Runs
7291 // only on a PROVEN commit (like every cleanup below): a recorded
7292 // post-commit error skips it, and the retry finishes the sweep.
7293 //
7294 // No directory fsync here, unlike the blob sweep below. Recovery replays
7295 // only the LIVE snapshot's log, so an entry a power loss resurrects is
7296 // never read; the next open recognizes it as an orphan log and sweeps it
7297 // again. Nothing observes the window, so the barrier would buy nothing.
7298 if post_commit_error.is_none() {
7299 let fresh_log = crate::version::edit_log::log_path(&config.path, version_id);
7300 match config.fs.read_dir(&config.path) {
7301 Ok(dirents) => {
7302 for dirent in dirents {
7303 if dirent.is_dir
7304 || !dirent.file_name.starts_with("edits-")
7305 || dirent.path == fresh_log
7306 {
7307 continue;
7308 }
7309 match config.fs.remove_file(&dirent.path) {
7310 Ok(()) => {}
7311 Err(e) if e.kind() == crate::io::ErrorKind::NotFound => {}
7312 Err(e) => {
7313 post_commit_error = Some(e.into());
7314 break;
7315 }
7316 }
7317 }
7318 }
7319 Err(e) => post_commit_error = Some(e.into()),
7320 }
7321 }
7322
7323 if let Some(p) = &config.recovery_progress {
7324 p.set_phase(crate::RecoveryPhase::Cleanup);
7325 }
7326
7327 // POST-COMMIT, step one: swap every finished replacement onto the name the
7328 // committed manifest gives it, destroying the damaged source it replaces.
7329 // Before this the manifest was not durable and the sources were the only
7330 // copies; after it the manifest describes exactly these bytes. A crash
7331 // between the commit and a swap leaves `{id}` damaged with `{id}.repair-tmp`
7332 // beside it — which the next run resolves from the committed manifest alone
7333 // (see `sweep_superseded_by_committed_manifest`), not from anything this run
7334 // remembered.
7335 //
7336 // NOT best-effort: the manifest already names this content, so a swap that
7337 // does not happen is a tree whose next open finds the damaged file under the
7338 // manifest's checksum and fails.
7339 //
7340 // The descriptors the repair cached close first: a file still open cannot
7341 // be replaced or removed on Windows, and nothing reads a held table again.
7342 // A tree opened on the shared cache meanwhile has its ids in the same range
7343 // and only loses cached descriptors, which it reopens.
7344 if let Some(descriptors) = &config.descriptor_table {
7345 descriptors.remove_trees_from(first_tree_id);
7346 }
7347 if post_commit_error.is_none() {
7348 for (fs, tmp_path, table_path, restricted) in swap_after_commit {
7349 if let Err(e) =
7350 commit_repair_tmp(&*fs, &tmp_path, &table_path, config.sync_mode, restricted)
7351 {
7352 log::error!(
7353 "repair: cannot swap the replacement {} onto {} ({e}); failing the \
7354 repair — the committed manifest names the replacement's content",
7355 tmp_path.display(),
7356 table_path.display(),
7357 );
7358 post_commit_error = Some(e);
7359 break;
7360 }
7361 }
7362 }
7363
7364 // POST-COMMIT, step two. The manifest is durable and names the salvaged
7365 // replacements, so the damaged originals they superseded are unreferenced
7366 // and are removed. This runs AFTER the commit on purpose: until then those
7367 // originals are what the tables the rewrite had not reached still point at,
7368 // so removing one earlier would let a failed repair leave a table
7369 // referencing a blob id that no longer exists — and the retry would then
7370 // record that table unrecoverable.
7371 //
7372 // NOT best-effort. A superseded original left in `blobs/` is outside the
7373 // committed manifest, so the next open classifies it as an orphan and
7374 // removes it — and if the directory refuses the removal now, that open
7375 // FAILS. Reporting a successful repair for a tree that will not open is the
7376 // one outcome recovery must never produce, so the failure propagates: the
7377 // manifest is already durable, and a retry once the filesystem is fixed
7378 // finishes the sweep on the same inputs. The report already names every
7379 // replacement; a removal only annotates its entry.
7380 let mut originals_removed = 0;
7381 if post_commit_error.is_none() {
7382 for (path, _) in &stale_blob_originals {
7383 match discard_unreferenced(&*config.fs, path, config.sync_mode) {
7384 Ok(()) => originals_removed += 1,
7385 Err(e) => {
7386 log::error!(
7387 "repair: cannot remove the superseded blob original {} ({e}); \
7388 failing the repair — left in blobs/ it is an orphan the next \
7389 open must remove, and that removal would hit the same error",
7390 path.display(),
7391 );
7392 post_commit_error = Some(e);
7393 break;
7394 }
7395 }
7396 }
7397 }
7398 // The originals' entries are the last ones of the list, in their order.
7399 for (i, (entry, (path, note))) in report
7400 .blob_files_salvaged
7401 .iter_mut()
7402 .skip(salvage_report_base)
7403 .zip(&stale_blob_originals)
7404 .enumerate()
7405 {
7406 let state = if i < originals_removed {
7407 "original removed"
7408 } else {
7409 "original NOT removed"
7410 };
7411 *entry = (path.clone(), format!("{note}; {state}"));
7412 }
7413
7414 // Everything the scan classified out of the rebuilt tree: a foreign name, a
7415 // duplicate id, a table or blob file no bound could make safe. The scan
7416 // itself touched nothing, so until this point a crash left the directory
7417 // exactly as the retry expects to find it; from here the manifest is
7418 // durable and these files are what the tree no longer references.
7419 if post_commit_error.is_none() {
7420 for (fs, path, note) in discard_after_commit {
7421 match discard_unreferenced(&*fs, &path, config.sync_mode) {
7422 Ok(()) => log::info!("repair: {} removed ({note})", path.display()),
7423 Err(e) => {
7424 log::error!(
7425 "repair: cannot remove {} ({e}); failing the repair — left in \
7426 place it is a file the next open must reject or sweep, and it \
7427 would hit the same error",
7428 path.display(),
7429 );
7430 post_commit_error = Some(e);
7431 break;
7432 }
7433 }
7434 }
7435 }
7436
7437 // Same rule for the blob files the manifest omits because nothing
7438 // references them: their data is unreachable, so there is nothing to
7439 // preserve — but leaving one behind hands the next open an orphan to
7440 // sweep, and a sweep that fails there fails the open. Remove them here and
7441 // propagate, so a repair that reports success leaves an openable tree.
7442 if post_commit_error.is_none() {
7443 let mut removed_dir: Option<std::path::PathBuf> = None;
7444 for path in unreferenced_blob_files {
7445 match config.fs.remove_file(&path) {
7446 Ok(()) => removed_dir = path.parent().map(std::path::Path::to_path_buf),
7447 Err(e) if e.kind() == crate::io::ErrorKind::NotFound => {}
7448 Err(e) => {
7449 log::error!(
7450 "repair: cannot remove the unreferenced blob file {} ({e}); \
7451 failing the repair — left in blobs/ it is an orphan the next \
7452 open must remove, and that removal would hit the same error",
7453 path.display(),
7454 );
7455 post_commit_error = Some(e.into());
7456 break;
7457 }
7458 }
7459 }
7460 // ONE sync for the batch, mirroring `remove_published_blob_replacements`:
7461 // the entries all live in the same directory, and without it a power
7462 // loss after this repair reports success can restore them, handing the
7463 // next open the very orphans these removals exist to prevent.
7464 if let Some(dir) = removed_dir
7465 && post_commit_error.is_none()
7466 && let Err(e) = config.fs.sync_directory_with(&dir, config.sync_mode)
7467 {
7468 log::error!(
7469 "repair: cannot make the removal of unreferenced blob files durable \
7470 in {} ({e}); failing the repair: a power loss would restore them \
7471 as orphans the next open must sweep",
7472 dir.display(),
7473 );
7474 post_commit_error = Some(e.into());
7475 }
7476 }
7477
7478 // A recorded post-commit failure carries the completed report out: the
7479 // manifest is durable, so the repair happened, and a retry once the
7480 // filesystem is fixed finishes the cleanup — but its own open may find
7481 // nothing left to repair and answer with no report at all.
7482 if let Some(cause) = post_commit_error {
7483 return Err(crate::Error::RepairedButUnopened {
7484 report: Box::new(report),
7485 cause: Box::new(cause),
7486 });
7487 }
7488
7489 // The rebuilt snapshot keeps owing the report until the caller
7490 // acknowledges it: a process that ends before the caller met the report's
7491 // replay obligation finds it there again.
7492
7493 // Success only: a failed run leaves the phase where it stopped, which
7494 // tells a progress display exactly which stage failed.
7495 if let Some(p) = &config.recovery_progress {
7496 p.set_phase(crate::RecoveryPhase::Done);
7497 }
7498
7499 Ok(report)
7500}
7501
7502#[cfg(feature = "std")]
7503mod report_marker;
7504
7505#[cfg(test)]
7506mod tests;