1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
//! What:
//! Owner inspection of one transaction registry entry,
//! and the oldest-first recovery order.
//! Why:
//! Live owners are skipped,
//! dead owners recovered,
//! and a published directory without a valid
//! owner record fails closed;
//! recovery runs oldest first because a later transaction was
//! prepared against the state an earlier one left
//! (`src/policy-engine/commit-transaction-recovery-inspect.ts`,
//! `commit-landing-lock.ts`).
//!
//! In TS you'd write (pseudocode):
//! ```ts
//! const inspected = oldestFirst(await Promise.all(entries.map(inspectRegistryEntry)));
//! ```
/// Debug diagnostics.
use super::diagnostic_log::debug;
/// Bounded no-follow reads.
use super::private_storage::{ReadRefusal, read_regular_file};
/// The fail-closed recovery failure.
use super::recovery_error::{RecoveryError, io_failure};
/// The recovery read limit and existence probes.
use super::recovery_files::{RECOVERY_FILE_LIMIT, recovery_path_exists};
/// Owner records and liveness.
use super::transaction_owner::{
OwnerLiveness, OwnerLivenessError, TransactionOwner, classify_transaction_owner,
parse_transaction_owner,
};
/// Registry entries.
use super::transaction_registry::{EntryKind, OWNER_FILENAME, RegistryEntry};
/// `PathBuf` is an owned filesystem path.
use std::path::PathBuf;
/// What:
/// An entry's owner evidence:
/// its record and liveness,
/// or why none can be attributed.
/// Why:
/// `InspectedEntry['owner']`.
///
/// In TS you'd write (pseudocode):
/// ```ts
/// type Owner = { record; liveness } | 'vanished' | 'unattributed';
/// ```
#[derive(Clone, Debug, Eq, PartialEq)]
pub enum OwnerEvidence {
/// A complete owner record and whether its process still runs.
Owned(TransactionOwner, OwnerLiveness),
/// The directory disappeared while recovery listed it:
/// its owner finished.
Vanished,
/// A staging candidate without a complete owner record.
Unattributed,
}
/// What:
/// One registry entry with its owner evidence.
/// Why:
/// `InspectedEntry`.
///
/// In TS you'd write (pseudocode):
/// ```ts
/// type InspectedEntry = { entry: TransactionRegistryEntry; owner: Owner };
/// ```
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct InspectedEntry {
/// The classified entry.
pub entry: RegistryEntry,
/// Its owner evidence.
pub owner: OwnerEvidence,
}
/// Methods over an inspected entry.
impl InspectedEntry {
/// What:
/// The owner record of a dead owner,
/// or nothing.
/// Why:
/// Only dead owners are recovered.
///
/// In TS you'd write (pseudocode):
/// ```ts
/// typeof owner !== 'string' && owner.liveness === 'dead' ? owner.record : undefined
/// ```
pub fn dead_owner(&self) -> Option<&TransactionOwner> {
match &self.owner {
OwnerEvidence::Owned(record, OwnerLiveness::Dead) => return Some(record),
_ => return None,
}
}
}
/// What:
/// Classify an owner,
/// reporting a failed probe as a recovery failure.
/// Why:
/// Recovery cannot decide liveness without the probe.
///
/// In TS you'd write (pseudocode):
/// ```ts
/// await classifyTransactionOwner({ ownerPid, ownerIdentity })
/// ```
fn liveness(record: &TransactionOwner) -> Result<OwnerLiveness, RecoveryError> {
match classify_transaction_owner(record.owner_pid, record.owner_identity.as_str()) {
Ok(found) => return Ok(found),
Err(OwnerLivenessError::Probe(error)) => {
return Err(RecoveryError(format!(
"probing transaction owner {} failed: {error}",
record.owner_pid
)));
}
Err(OwnerLivenessError::Identity(error)) => {
return Err(RecoveryError(format!(
"reading the identity of transaction owner {} failed: {error}",
record.owner_pid
)));
}
}
}
/// What:
/// Read and classify an entry's owner,
/// tolerating a live owner removing its own directory.
/// Why:
/// `inspectRegistryEntry`:
/// a published directory needs a valid matching owner record;
/// a
/// staging candidate may hold a torn one.
///
/// In TS you'd write (pseudocode):
/// ```ts
/// async function inspectRegistryEntry(entry: TransactionRegistryEntry): Promise<InspectedEntry>;
/// ```
pub fn inspect_registry_entry(entry: &RegistryEntry) -> Result<InspectedEntry, RecoveryError> {
let owner_path: PathBuf = entry.path.join(OWNER_FILENAME);
let staging: bool = entry.kind == EntryKind::Staging;
let bytes: Vec<u8> = match read_regular_file(owner_path.as_path(), RECOVERY_FILE_LIMIT) {
Ok(found) => found,
Err(ReadRefusal::Missing) => {
debug(
"inspectRegistryEntry",
format!("owner record absent at {}", owner_path.display()).as_str(),
);
if staging {
return Ok(unattributed(entry));
}
if !recovery_path_exists(entry.path.as_path())? {
return Ok(InspectedEntry {
entry: entry.clone(),
owner: OwnerEvidence::Vanished,
});
}
return Err(RecoveryError(format!(
"Transaction owner record is missing: {}",
entry.path.display()
)));
}
Err(ReadRefusal::NotRegular | ReadRefusal::TooLarge) => {
return Err(RecoveryError(format!(
"Unsafe transaction recovery file: {}",
owner_path.display()
)));
}
Err(ReadRefusal::Io(error)) => {
return Err(io_failure("reading", owner_path.as_path(), &error));
}
};
let Some(record) = parse_transaction_owner(bytes.as_slice()) else {
if staging {
debug(
"inspectRegistryEntry",
format!("staging owner record incomplete: {}", entry.path.display()).as_str(),
);
return Ok(unattributed(entry));
}
return Err(RecoveryError(format!(
"Transaction owner record is malformed: {}",
entry.path.display()
)));
};
if record.transaction_id != entry.transaction_id {
return Err(RecoveryError(format!(
"Transaction owner record names another transaction: {}",
entry.path.display()
)));
}
let state: OwnerLiveness = liveness(&record)?;
return Ok(InspectedEntry {
entry: entry.clone(),
owner: OwnerEvidence::Owned(record, state),
});
}
/// What:
/// An entry whose owner cannot be attributed.
/// Why:
/// Shared by the missing and torn staging cases.
///
/// In TS you'd write (pseudocode):
/// ```ts
/// ({ entry, owner: 'unattributed' })
/// ```
fn unattributed(entry: &RegistryEntry) -> InspectedEntry {
return InspectedEntry {
entry: entry.clone(),
owner: OwnerEvidence::Unattributed,
};
}
/// What:
/// An entry's recovery-order key:
/// `<createdAt> <path>` when attributed,
/// otherwise
/// `~<path>`,
/// after every ISO time.
/// Why:
/// `creationKey`.
///
/// In TS you'd write (pseudocode):
/// ```ts
/// typeof owner === 'string' ? `~${entry.path}` : `${owner.record.createdAt} ${entry.path}`
/// ```
pub fn creation_key(inspected: &InspectedEntry) -> String {
let path: String = inspected.entry.path.to_string_lossy().into_owned();
match &inspected.owner {
OwnerEvidence::Owned(record, _) => return format!("{} {path}", record.created_at),
OwnerEvidence::Vanished | OwnerEvidence::Unattributed => return format!("~{path}"),
}
}
/// What:
/// Order inspected entries oldest first,
/// stably.
/// Why:
/// `oldestFirst`:
/// JavaScript compares the keys by UTF-16 code units;
/// for the ASCII
/// timestamps and registry paths the wrappers write,
/// byte order is the same order.
///
/// In TS you'd write (pseudocode):
/// ```ts
/// entries.toSorted((left, right) => compare(creationKey(left), creationKey(right)))
/// ```
pub fn oldest_first(entries: Vec<InspectedEntry>) -> Vec<InspectedEntry> {
// `mut` allows sorting the keyed entries in place.
let mut keyed: Vec<(String, InspectedEntry)> = Vec::with_capacity(entries.len());
for entry in entries {
keyed.push((creation_key(&entry), entry));
}
keyed.sort_by(compare_keys);
let mut ordered: Vec<InspectedEntry> = Vec::with_capacity(keyed.len());
for (_, entry) in keyed {
ordered.push(entry);
}
return ordered;
}
/// What:
/// Compare two keyed entries by key.
/// Why:
/// A named comparator keeps the sort free of closures.
///
/// In TS you'd write (pseudocode):
/// ```ts
/// (left, right) => left.key < right.key ? -1 : left.key > right.key ? 1 : 0
/// ```
fn compare_keys(
left: &(String, InspectedEntry),
right: &(String, InspectedEntry),
) -> std::cmp::Ordering {
return left.0.cmp(&right.0);
}
/// Inspection controls stay out of the release executable.
#[cfg(test)]
#[path = "recovery_inspect_tests.rs"]
pub(crate) mod tests;