nodedb_wal/writer/
dwb.rs

1// SPDX-License-Identifier: Apache-2.0
2
3//! The writer's side of the double-write buffer: mirroring records into it,
4//! and keeping the loss of torn-write protection visible when it fails.
5//!
6//! A DWB problem deliberately does NOT fail the WAL write that provoked it —
7//! the WAL append is correct on its own and is still acknowledged, and turning
8//! a degraded mode into an outage would be a worse trade. What a DWB problem
9//! does cost is recoverability: a record whose CRC fails with no DWB copy
10//! behind it is indistinguishable from the end of the committed prefix, so a
11//! torn last record in a segment is dropped in silence. That cost is recorded
12//! on the writer and in the process counters instead of being logged once and
13//! forgotten.
14
15use crate::double_write::{DwbDegradation, DwbMirror, DwbProtection, DwbSkipReason, metrics};
16use crate::record::WalRecord;
17
18use super::config::{open_dwb_at, resolve_dwb_mode};
19use super::core::WalWriter;
20
21impl WalWriter {
22    /// Torn-write protection standing of this writer.
23    pub fn dwb_protection(&self) -> DwbProtection {
24        self.dwb_protection
25    }
26
27    /// Records appended to this segment with no double-write copy behind them.
28    ///
29    /// Non-zero means those records cannot be reconstructed if they are torn
30    /// by a power loss at the tail of the segment.
31    pub fn dwb_unprotected_records(&self) -> u64 {
32        self.dwb_unprotected_records
33    }
34
35    /// Reopen a double-write buffer that was detached after a failure.
36    ///
37    /// Detaching keeps a broken device from being hammered once per append,
38    /// but on its own it makes degradation permanent for the life of the
39    /// writer — an operator who has fixed the fault would have no way back
40    /// short of rolling the segment. Returns the resulting standing.
41    pub fn reattach_double_write(&mut self) -> DwbProtection {
42        let Some(dwb_path) = self.dwb_path.clone() else {
43            return self.dwb_protection;
44        };
45        match open_dwb_at(&dwb_path, resolve_dwb_mode(&self.config)) {
46            Some(dwb) => {
47                self.double_write = Some(dwb);
48                self.dwb_protection = DwbProtection::Active;
49            }
50            None => self.degrade_dwb(DwbDegradation::OpenFailed),
51        }
52        self.dwb_protection
53    }
54
55    /// Mirror a record that is already committed to the WAL write buffer.
56    pub(super) fn mirror_into_dwb(&mut self, lsn: u64, record: &WalRecord) {
57        let mirrored = self
58            .double_write
59            .as_mut()
60            .map(|dwb| dwb.write_record_deferred(record));
61        match mirrored {
62            Some(Ok(DwbMirror::Mirrored)) => {}
63            Some(Ok(DwbMirror::Skipped(reason))) => self.note_unprotected_record(lsn, reason),
64            Some(Err(e)) => {
65                tracing::warn!(lsn = lsn, error = %e, "DWB write failed, detaching DWB");
66                self.double_write = None;
67                self.degrade_dwb(DwbDegradation::WriteFailed);
68                self.note_unprotected();
69            }
70            // No buffer attached: either the DWB is off by configuration
71            // (nothing to report) or it already degraded, in which case every
72            // record since is unprotected and has to keep being counted.
73            None => {
74                if self.dwb_protection.is_degraded() {
75                    self.note_unprotected();
76                }
77            }
78        }
79    }
80
81    /// Make the mirrored batch durable ahead of the WAL's own fsync.
82    ///
83    /// The staged slots are written first. A failed slot write detaches the
84    /// DWB as a failed record write does, and every staged record that would
85    /// have reached a slot is counted unprotected.
86    ///
87    /// A failed DWB fsync leaves the batch mirrored only in the page cache, so
88    /// its records are unprotected even though they reached slots. The WAL
89    /// sync continues regardless — the WAL's own fsync is what decides
90    /// durability — and the lost protection is recorded.
91    pub(super) fn flush_dwb(&mut self) {
92        let staged = self
93            .double_write
94            .as_ref()
95            .map_or(0, |dwb| dwb.staged_slots());
96        let written = self.double_write.as_mut().map(|dwb| dwb.write_staged());
97        if let Some(Err(e)) = written {
98            tracing::warn!(error = %e, staged, "DWB slot write failed, detaching DWB");
99            self.double_write = None;
100            self.degrade_dwb(DwbDegradation::WriteFailed);
101            for _ in 0..staged {
102                self.note_unprotected();
103            }
104            return;
105        }
106        let flushed = self.double_write.as_mut().map(|dwb| dwb.flush());
107        if let Some(Err(e)) = flushed {
108            tracing::warn!(error = %e, "DWB flush failed, detaching DWB");
109            self.double_write = None;
110            self.degrade_dwb(DwbDegradation::FlushFailed);
111        }
112    }
113
114    /// Account for a record the WAL accepted but the DWB could not mirror.
115    fn note_unprotected_record(&mut self, lsn: u64, reason: DwbSkipReason) {
116        tracing::warn!(
117            lsn = lsn,
118            reason = %reason,
119            "record appended without torn-write protection"
120        );
121        self.note_unprotected();
122    }
123
124    fn note_unprotected(&mut self) {
125        self.dwb_unprotected_records = self.dwb_unprotected_records.saturating_add(1);
126        metrics::add_unprotected_record();
127    }
128
129    /// Record that protection was requested and has now been lost.
130    fn degrade_dwb(&mut self, degradation: DwbDegradation) {
131        if !self.dwb_protection.is_degraded() {
132            metrics::add_degradation();
133        }
134        self.dwb_protection = DwbProtection::Degraded(degradation);
135    }
136}
nodedb_wal/writer/dwb.rs

nodedb_wal/writer/
dwb.rs
