gunnar_core/audit.rs
1//! **The control plane's audit trail, made tamper-evident** — an append-only
2//! hash chain over the decisions the gRPC gate makes.
3//!
4//! # What was there, and why it was not evidence
5//!
6//! Every authorised RPC already emitted one `tracing` record on
7//! `gunnar::audit` — method, principal, key fingerprint, scope, level, decision.
8//! That is a good *log* and it is not a *trail*: it goes wherever the
9//! subscriber points it, usually a file or a journal that the same box's root
10//! can rewrite, and nothing in it says whether it has been rewritten. An
11//! operator reading it cannot tell a complete record from one a line was cut
12//! out of, which is exactly the question an audit trail exists to answer. The
13//! security review recorded it as GUN-044.
14//!
15//! # The shape: each entry names the one before it
16//!
17//! ```text
18//! entry.hash = SHA-256( "gunnar-audit-v1" ‖ prev ‖ len‖field … )
19//! ```
20//!
21//! The first entry's `prev` is [`GENESIS`] (64 zeros). Every later entry's
22//! `prev` is the previous entry's `hash`. So the tip hash commits to the whole
23//! history: change one byte of one entry and that entry's hash changes, which
24//! breaks the `prev` of the one after it, and so on to the end. Removing an
25//! entry breaks the same link; so does reordering two. There is no way to edit
26//! the middle of the file without rewriting every line after it, and a
27//! recorded tip (an operator's note, a second box, a monitoring system that
28//! reads the tip once a day) makes even that detectable.
29//!
30//! **This is not the transparency log.** [`crate::transparency`] is a Merkle
31//! tree with signed heads and inclusion proofs, which is the right shape for a
32//! log a *third party* audits without trusting the server. This is a linear
33//! chain with no signature, which is the right shape for a trail an *operator*
34//! checks on their own box, and it costs one hash per RPC and no key
35//! management. Making the control-plane trail publicly verifiable is a
36//! different, larger decision; this one closes the finding that was actually
37//! reported — a trail that cannot say whether it is complete.
38//!
39//! # The fields are length-prefixed, and that is the load-bearing detail
40//!
41//! A chain whose fields were joined with a separator would let a value
42//! containing that separator move the boundary: a `reason` of `"x\nallow"`
43//! could hash identically to a different entry whose decision really was
44//! `allow`. So each field goes in as its byte length (8 bytes, little-endian)
45//! followed by its bytes, which is unambiguous whatever the field holds. The
46//! same reasoning as [`crate::transparency`]'s domain-separating prefixes, one
47//! layer down.
48//!
49//! # One file, appended, fsynced
50//!
51//! JSON Lines, one entry per line, beside the rest of the control state — the
52//! same choice push attestation already made, for the same reason: a record
53//! that outlives the process must be readable by something that is not gunnar.
54//! Each append is durable before it returns. On the primary's REPLICATED trail
55//! ([`AuditChain::open_framed`]) the durable copy is the `Append` frame in the
56//! transaction log — group-committed with every frame staged beside it
57//! (perf, 2026-10-06: concurrent decisions share one `fdatasync`, measured
58//! 2.0 → 0.26 flushes per decision on eight request threads) — and the line in
59//! this file follows it without an fsync of its own: the file is the view of the
60//! log's frames that every start checks and rebuilds
61//! (`gunnar_server::twin::rematerialise_audit_trail`). An unframed chain (a
62//! twin's own trail) `sync_all`s each line as before.
63//!
64//! How decisions reach the chain is the gate's ledger's call
65//! (`gunnar_server::grpc_gate`): every refusal, write, admin call, self-scope
66//! call and override is one entry at once; repeated plain READS are aggregated —
67//! the first of a 60 s window at once, then one entry with the count and the
68//! first and last time (P3 of the load test, 2026-10-06, `READ_WINDOW`).
69
70use std::io::Write as _;
71use std::path::{Path, PathBuf};
72use std::sync::Mutex;
73
74use sha2::Digest as _;
75
76/// The file the chain lives in, under the control directory.
77pub const AUDIT_FILE: &str = "audit-chain.jsonl";
78
79/// Domain separation, so a hash from this chain is not a hash from anything
80/// else in this repository that also runs SHA-256 over records.
81const DOMAIN: &str = "gunnar-audit-v1";
82
83/// What the first entry's `prev` is: 64 zeros. A literal rather than an empty
84/// string, so every entry in the file has the same shape and a truncated-to-
85/// nothing `prev` cannot be mistaken for the start of the chain.
86pub const GENESIS: &str = "0000000000000000000000000000000000000000000000000000000000000000";
87
88/// What went wrong with the chain.
89#[derive(Debug, thiserror::Error)]
90pub enum AuditError {
91 /// The file could not be read, created or written.
92 #[error("the audit chain at {path}: {source}")]
93 Io {
94 /// Which file.
95 path: String,
96 /// The underlying failure.
97 source: std::io::Error,
98 },
99 /// A line in the file is not an entry.
100 #[error("the audit chain entry on line {line} does not parse: {source}")]
101 Malformed {
102 /// 1-based line number.
103 line: usize,
104 /// What serde said.
105 source: serde_json::Error,
106 },
107 /// The line could not be framed into the transaction log, so it was not
108 /// written down at all. **Deliberately not best-effort**: a trail whose
109 /// file and whose log disagree is worse than one missing entry, because
110 /// the missing entry is announced and the disagreement is not.
111 #[error("the audit chain at {path} could not be framed as a transaction: {detail}")]
112 Frame {
113 /// Which file.
114 path: String,
115 /// What the log said.
116 detail: String,
117 },
118 /// This chain was asked to be the REPLICATED trail and cannot be
119 /// ([`AuditChain::open_framed`]). Never silent: the caller says so out
120 /// loud and opens an unframed chain instead.
121 #[error("the audit chain at {path} cannot be the replicated trail: {why}")]
122 Unframed {
123 /// Which file.
124 path: String,
125 /// Why not, in a sentence an operator can act on.
126 why: String,
127 },
128 /// The box FOLLOWS (a twin, or a box being restored): it originates nothing into the
129 /// replicated trail it holds for its primary, so nothing was written ([`Recorded::NotOriginated`]
130 /// through [`AuditChain::append`]). Not a fault: the caller's log line is the record.
131 #[error("the audit chain at {path} is the primary's trail and this box follows it (a twin, or restoring): it originates nothing into it")]
132 NotOriginated {
133 /// Which file.
134 path: String,
135 },
136 /// The chain does not hold. **This is the finding, not a fault.**
137 #[error("the audit chain is broken at entry {seq} (line {line}): {what}")]
138 Broken {
139 /// The `seq` the entry claims.
140 seq: u64,
141 /// 1-based line number.
142 line: usize,
143 /// Which invariant failed, in a sentence an operator can act on.
144 what: String,
145 },
146}
147
148/// One decision, as the control plane hands it over.
149///
150/// Borrowed, because every field of it is already a `&str` at the call site and
151/// an audit record must not make the decision path allocate.
152#[derive(Debug, Clone, Copy)]
153pub struct AuditEvent<'a> {
154 /// `Service.Method`, e.g. `Access.Grant`.
155 pub method: &'a str,
156 /// The ACL principal id the credential resolved to.
157 pub principal: &'a str,
158 /// `SHA256:…` of the key that was presented, empty under the development
159 /// escape hatch.
160 pub fingerprint: &'a str,
161 /// What was asked about: `server`, `namespace:x`, `repo:x/y`.
162 pub scope: &'a str,
163 /// The grant level the method needs.
164 pub level: &'a str,
165 /// `required` or `allow-all-insecure`.
166 pub auth_mode: &'a str,
167 /// `allow`, `deny` or `narrow` — the same three words the `tracing` record
168 /// carries, so the two can be read side by side.
169 pub decision: &'a str,
170 /// Why, for a decision that has a why. Empty otherwise.
171 pub reason: &'a str,
172}
173
174/// One entry, as it sits on disk.
175///
176/// Field order is the serialisation order and the hashing order. It is
177/// `deny_unknown_fields` so a file written by a newer gunnar is refused loudly
178/// rather than verified against fields this build cannot see.
179#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
180#[serde(deny_unknown_fields)]
181pub struct AuditEntry {
182 /// 0 for the first entry, then +1 each time. Redundant with the file's line
183 /// numbering on purpose: a chain whose only ordering was its line order
184 /// could be truncated to a shorter, still-valid chain without the entries
185 /// themselves saying anything was missing.
186 pub seq: u64,
187 /// When the decision was made.
188 pub at_unix_ms: i64,
189 /// `Service.Method`.
190 pub method: String,
191 /// The ACL principal id.
192 pub principal: String,
193 /// The key fingerprint.
194 pub fingerprint: String,
195 /// The scope asked about.
196 pub scope: String,
197 /// The grant level.
198 pub level: String,
199 /// The authentication mode the listener was in.
200 pub auth_mode: String,
201 /// `allow`, `deny`, `narrow`.
202 pub decision: String,
203 /// Why, or empty.
204 pub reason: String,
205 /// The previous entry's [`AuditEntry::hash`], or [`GENESIS`].
206 pub prev: String,
207 /// This entry's own hash. Not hashed into itself; see [`AuditEntry::compute_hash`].
208 pub hash: String,
209}
210
211impl AuditEntry {
212 /// The hash this entry **should** carry, computed from everything else in
213 /// it.
214 ///
215 /// Verification is `entry.hash == entry.compute_hash()` plus
216 /// `entry.prev == previous.hash`. Both halves are needed: the first catches
217 /// an edited field, the second catches a removed, reordered or inserted
218 /// entry.
219 pub fn compute_hash(&self) -> String {
220 let mut hasher = sha2::Sha256::new();
221 let mut field = |bytes: &[u8]| {
222 // Length-prefixed, so no value can move a field boundary. See the
223 // module docs.
224 hasher.update((bytes.len() as u64).to_le_bytes());
225 hasher.update(bytes);
226 };
227 field(DOMAIN.as_bytes());
228 field(self.prev.as_bytes());
229 field(&self.seq.to_le_bytes());
230 field(&self.at_unix_ms.to_le_bytes());
231 field(self.method.as_bytes());
232 field(self.principal.as_bytes());
233 field(self.fingerprint.as_bytes());
234 field(self.scope.as_bytes());
235 field(self.level.as_bytes());
236 field(self.auth_mode.as_bytes());
237 field(self.decision.as_bytes());
238 field(self.reason.as_bytes());
239 let digest = hasher.finalize();
240 let mut out = String::with_capacity(64);
241 for byte in digest {
242 use std::fmt::Write as _;
243 let _ = write!(out, "{byte:02x}");
244 }
245 out
246 }
247}
248
249/// **The chain itself.** Open it once and append to it; it is `Send + Sync` and
250/// holds its own lock, because one server writes one chain from many request
251/// threads and two appends that interleaved would each claim the same `prev`.
252#[derive(Debug)]
253pub struct AuditChain {
254 path: PathBuf,
255 tip: Mutex<Tip>,
256 /// **Is every entry also a transaction?** ([`AuditChain::open_framed`]).
257 ///
258 /// The primary's trail is: it goes into the box's transaction log as an
259 /// `Append` frame, so the twin holds it and a restore brings it back. The
260 /// twin's OWN trail is not, and neither is a test's — see
261 /// [`crate::txlog::AUDIT_LOCAL_REL`].
262 framed: bool,
263}
264
265/// What the next entry needs to know about the last one.
266#[derive(Debug)]
267struct Tip {
268 next_seq: u64,
269 hash: String,
270 /// Lines computed while the log was RELEASED (the data set being grown and unmounted),
271 /// oldest first, each already linked to the one before it: framed and written, in order,
272 /// once the log is taken back ([`AuditChain::flush`]).
273 queued: Vec<String>,
274 /// The box followed its primary since this tip was read: the file may hold the primary's
275 /// entries now, so the tip is read from the file again before the next write.
276 stale: bool,
277 /// **Lines framed and not yet in the file**, oldest first, each with its frame's sequence
278 /// number (perf, 2026-10-06): a framed chain stages its frame, waits for the group's fsync
279 /// with the tip unlocked — so concurrent decisions share one — and only then writes the line,
280 /// in chain order, every line whose frame is durable.
281 unfiled: Vec<(u64, String)>,
282}
283
284/// **What became of one decision** ([`AuditChain::record`]).
285#[derive(Debug, Clone)]
286pub enum Recorded {
287 /// In the file (and, for a framed chain, in the transaction log).
288 Written(AuditEntry),
289 /// The log is RELEASED (the data set is being grown): the entry is linked and held in
290 /// memory, in order, and is framed and written by the first [`AuditChain::record`] or
291 /// [`AuditChain::flush`] after the remount. `pending` counts the held entries.
292 Queued {
293 /// The entry, as it will be written.
294 entry: AuditEntry,
295 /// How many entries wait.
296 pending: usize,
297 },
298 /// The box FOLLOWS its primary (a twin, or restoring): a framed chain is the primary's
299 /// trail, and this box originates nothing into it. Written nowhere.
300 NotOriginated,
301}
302
303/// What one framed write came to.
304enum Wrote {
305 Yes,
306 /// The log is released: nothing framed, nothing written.
307 Released,
308 /// The log follows: nothing framed, nothing written.
309 Follower,
310}
311
312impl AuditChain {
313 /// Open — or start — the chain at `path`.
314 ///
315 /// The existing file is **read and verified** at open, not merely counted.
316 /// A server that appended to a chain it had not checked would extend
317 /// somebody's forgery with genuine entries and make the break look like it
318 /// happened later than it did.
319 pub fn open(path: impl Into<PathBuf>) -> Result<AuditChain, AuditError> {
320 let path = path.into();
321 if let Some(parent) = path.parent() {
322 std::fs::create_dir_all(parent).map_err(|source| AuditError::Io {
323 path: parent.display().to_string(),
324 source,
325 })?;
326 }
327 let entries = read_entries(&path)?;
328 verify_entries(&entries)?;
329 let tip = match entries.last() {
330 Some(last) => Tip {
331 next_seq: last.seq + 1,
332 hash: last.hash.clone(),
333 queued: Vec::new(),
334 stale: false,
335 unfiled: Vec::new(),
336 },
337 None => Tip {
338 next_seq: 0,
339 hash: GENESIS.to_owned(),
340 queued: Vec::new(),
341 stale: false,
342 unfiled: Vec::new(),
343 },
344 };
345 Ok(AuditChain {
346 path,
347 tip: Mutex::new(tip),
348 framed: false,
349 })
350 }
351
352 /// **Open the chain as the REPLICATED trail** — every append is an
353 /// `Append` transaction before it is a line in a file, so the twin holds
354 /// the trail and `gunnar control twin replay` brings it back.
355 ///
356 /// This is the D6 fix (lane T11, 2026-09-21). Before it, `log/` was a
357 /// per-box path: the twin never held the audit trail and a restore came up
358 /// with none, reporting success. The restore happens on the worst day,
359 /// which is the day somebody has to reconstruct what happened.
360 ///
361 /// **It refuses rather than falling back.** The chain must sit at
362 /// [`crate::txlog::AUDIT_REL`] under a data root the sink knows, or this
363 /// returns [`AuditError::Unframed`] naming why. A chain that quietly
364 /// wrote unframed lines would reproduce the exact bug — a trail the
365 /// restore drops without saying so. The caller's job is to say it out
366 /// loud and open an ordinary chain with [`AuditChain::open`].
367 pub fn open_framed(path: impl Into<PathBuf>) -> Result<AuditChain, AuditError> {
368 let path = path.into();
369 match crate::txlog::sink::log_for(&path) {
370 None => {
371 return Err(AuditError::Unframed {
372 path: path.display().to_string(),
373 why: "it is under no data root that has a transaction log — the layout put \
374 the `log` directory outside the data root, or this box originates \
375 nothing (a twin, a box being restored)"
376 .to_owned(),
377 })
378 }
379 Some((_, rel)) if rel != crate::txlog::AUDIT_REL => {
380 return Err(AuditError::Unframed {
381 path: path.display().to_string(),
382 why: format!(
383 "the log would name it {rel:?}, and a restore resolves only {:?}",
384 crate::txlog::AUDIT_REL
385 ),
386 })
387 }
388 Some(_) => {}
389 }
390 let mut chain = AuditChain::open(path)?;
391 chain.framed = true;
392 Ok(chain)
393 }
394
395 /// Does every append of this chain also become a transaction? See
396 /// [`AuditChain::open_framed`].
397 pub fn is_framed(&self) -> bool {
398 self.framed
399 }
400
401 /// Where the chain lives.
402 pub fn path(&self) -> &Path {
403 &self.path
404 }
405
406 /// **Append one decision.** Returns the entry that was written (or, while the log is
407 /// released, queued — [`Recorded::Queued`]), whose `hash` is the new tip an operator can
408 /// record. A box that follows its primary answers [`AuditError::NotOriginated`].
409 pub fn append(&self, event: AuditEvent<'_>, at_unix_ms: i64) -> Result<AuditEntry, AuditError> {
410 match self.record(event, at_unix_ms)? {
411 Recorded::Written(entry) | Recorded::Queued { entry, .. } => Ok(entry),
412 Recorded::NotOriginated => Err(AuditError::NotOriginated {
413 path: self.path.display().to_string(),
414 }),
415 }
416 }
417
418 /// The log this chain frames into, when it is framed and the sink knows its root.
419 fn framing_log(&self) -> Option<std::sync::Arc<crate::txlog::Log>> {
420 if self.framed {
421 crate::txlog::sink::log_for(&self.path).map(|(log, _)| log)
422 } else {
423 None
424 }
425 }
426
427 /// **Record one decision** — [`Recorded`] says what became of it.
428 ///
429 /// G + G2 (release-3 test2 mock 2026-09-29): a chain opened framed on a box that was
430 /// then told `BecomeTwin` at runtime, and a primary whose log was released for its own
431 /// growth, answered every decision with an error (`txlog: this box is a twin …`, `txlog:
432 /// released …`) that the gate logged at ERROR — 13 and 4 lines per mock — and the grow's
433 /// decisions were lost. A follower now originates nothing and says so without an error; a
434 /// released log holds the trail in memory, in order, and writes it after the remount.
435 pub fn record(&self, event: AuditEvent<'_>, at_unix_ms: i64) -> Result<Recorded, AuditError> {
436 // A framed decision is a mutation of the replicated trail, and its guard is taken BEFORE
437 // the chain's own lock and held until the file has the line: a snapshot's freeze waits
438 // for every guard in flight, and a thread that held the chain while it waited on the
439 // freeze would stop the very thread the freeze waits for (that one files its line under
440 // the chain's lock after its frame's fsync). The guards the frame write takes inside are
441 // nested and count once.
442 let _mutation = self.framing_log().map(|log| log.mutation());
443 // Poisoning is not a reason to stop writing the audit trail: a panic in
444 // some other request's append leaves the tip behind, and refusing every
445 // later record would turn one bug into a gap in the evidence.
446 let mut tip = self
447 .tip
448 .lock()
449 .unwrap_or_else(|poisoned| poisoned.into_inner());
450 let log = self.framing_log();
451 if log.as_ref().is_some_and(|l| l.is_follower()) {
452 // The file is the PRIMARY's trail now (the twin applies its frames into it): a
453 // local line would splice a second writer into a one-writer chain. Anything held
454 // from before is the primary role's, which ended.
455 tip.queued.clear();
456 tip.unfiled.clear();
457 tip.stale = true;
458 return Ok(Recorded::NotOriginated);
459 }
460 if tip.stale {
461 // Promoted again: continue from what the file holds NOW, not from the tip this
462 // box remembered before it followed.
463 let entries = read_entries(&self.path)?;
464 match entries.last() {
465 Some(last) => {
466 tip.next_seq = last.seq + 1;
467 tip.hash.clone_from(&last.hash);
468 }
469 None => {
470 tip.next_seq = 0;
471 tip.hash = GENESIS.to_owned();
472 }
473 }
474 tip.stale = false;
475 }
476 let released = log.as_ref().is_some_and(|l| l.is_released());
477 if !released {
478 self.flush_locked(&mut tip)?;
479 }
480
481 let mut entry = AuditEntry {
482 seq: tip.next_seq,
483 at_unix_ms,
484 method: event.method.to_owned(),
485 principal: event.principal.to_owned(),
486 fingerprint: event.fingerprint.to_owned(),
487 scope: event.scope.to_owned(),
488 level: event.level.to_owned(),
489 auth_mode: event.auth_mode.to_owned(),
490 decision: event.decision.to_owned(),
491 reason: event.reason.to_owned(),
492 prev: tip.hash.clone(),
493 hash: String::new(),
494 };
495 entry.hash = entry.compute_hash();
496
497 let line = serde_json::to_string(&entry).expect("an audit entry serialises");
498 debug_assert!(!line.contains('\n'), "a JSONL record holds no newline");
499
500 if released || !tip.queued.is_empty() {
501 return Ok(Self::hold(&mut tip, entry, line));
502 }
503 if self.framed {
504 // ── the group-committed path (perf, 2026-10-06) ──
505 match crate::txlog::sink::stage_line(&self.path, &line) {
506 Ok(Some(staged)) => {
507 tip.next_seq = entry.seq + 1;
508 tip.hash.clone_from(&entry.hash);
509 tip.unfiled.push((staged.seq(), line));
510 // The fsync is waited for with the tip UNLOCKED: decisions made meanwhile
511 // link onto this one and stage their frames behind it, and one leader's
512 // fdatasync makes them all durable.
513 drop(tip);
514 staged.durable().map_err(|e| AuditError::Frame {
515 path: self.path.display().to_string(),
516 detail: e.to_string(),
517 })?;
518 let mut tip = self
519 .tip
520 .lock()
521 .unwrap_or_else(|poisoned| poisoned.into_inner());
522 self.file_durable(&mut tip, staged.log())?;
523 drop(tip);
524 // The mutation guard goes only now: the file has the line.
525 drop(staged);
526 return Ok(Recorded::Written(entry));
527 }
528 Ok(None) => {} // under no registered root any more: the unframed write below
529 Err(crate::txlog::Error::Released(_)) => {
530 return Ok(Self::hold(&mut tip, entry, line))
531 }
532 Err(crate::txlog::Error::Follower) => {
533 tip.stale = true;
534 return Ok(Recorded::NotOriginated);
535 }
536 Err(source) => {
537 return Err(AuditError::Frame {
538 path: self.path.display().to_string(),
539 detail: source.to_string(),
540 });
541 }
542 }
543 }
544 match self.write_line(&line)? {
545 Wrote::Yes => {}
546 Wrote::Released => return Ok(Self::hold(&mut tip, entry, line)),
547 Wrote::Follower => {
548 tip.stale = true;
549 return Ok(Recorded::NotOriginated);
550 }
551 }
552 tip.next_seq = entry.seq + 1;
553 tip.hash.clone_from(&entry.hash);
554 Ok(Recorded::Written(entry))
555 }
556
557 /// Hold a linked entry until the log is taken back.
558 fn hold(tip: &mut Tip, entry: AuditEntry, line: String) -> Recorded {
559 tip.queued.push(line);
560 tip.next_seq = entry.seq + 1;
561 tip.hash.clone_from(&entry.hash);
562 Recorded::Queued {
563 entry,
564 pending: tip.queued.len(),
565 }
566 }
567
568 /// How many entries wait for the log to be taken back ([`Recorded::Queued`]).
569 pub fn queued(&self) -> usize {
570 self.tip
571 .lock()
572 .unwrap_or_else(|poisoned| poisoned.into_inner())
573 .queued
574 .len()
575 }
576
577 /// **Write what waits** — every held entry, in order, once the log is taken back (the
578 /// remount after a growth). Answers how many were written; a log still released writes
579 /// none and keeps them.
580 pub fn flush(&self) -> Result<usize, AuditError> {
581 let mut tip = self
582 .tip
583 .lock()
584 .unwrap_or_else(|poisoned| poisoned.into_inner());
585 if self
586 .framing_log()
587 .is_some_and(|l| l.is_released() || l.is_follower())
588 {
589 return Ok(0);
590 }
591 self.flush_locked(&mut tip)
592 }
593
594 /// **Write every framed line whose frame is durable into the file**, oldest first, as ONE
595 /// write and WITHOUT an fsync: the frame is the durable copy, and the file is the view of the
596 /// log's frames that every start checks and rebuilds (`gunnar_server::twin::
597 /// rematerialise_audit_trail`, T17) — an fsync per line here bought nothing the log does not
598 /// already hold, at the cost of a second flush per decision. A line whose write fails stays
599 /// unfiled and goes with the next.
600 fn file_durable(&self, tip: &mut Tip, log: &crate::txlog::Log) -> Result<usize, AuditError> {
601 let head = log.head_seq();
602 let n = tip
603 .unfiled
604 .iter()
605 .take_while(|(seq, _)| *seq <= head)
606 .count();
607 if n == 0 {
608 return Ok(0);
609 }
610 let mut bytes = Vec::with_capacity(tip.unfiled[..n].iter().map(|(_, l)| l.len() + 1).sum());
611 for (_, l) in &tip.unfiled[..n] {
612 bytes.extend_from_slice(l.as_bytes());
613 bytes.push(b'\n');
614 }
615 let io = |source: std::io::Error| AuditError::Io {
616 path: self.path.display().to_string(),
617 source,
618 };
619 let mut file = std::fs::OpenOptions::new()
620 .create(true)
621 .append(true)
622 .open(&self.path)
623 .map_err(io)?;
624 file.write_all(&bytes).map_err(io)?;
625 tip.unfiled.drain(..n);
626 Ok(n)
627 }
628
629 fn flush_locked(&self, tip: &mut Tip) -> Result<usize, AuditError> {
630 // Framed lines whose frames are durable go into the file first — they precede anything
631 // held. Their fsync is NOT forced here (that would hold the tip across an fsync and undo
632 // the group commit): the thread that staged each one files it. Only held lines waiting
633 // behind them (a release just ended) are worth an fsync of their own.
634 if !tip.unfiled.is_empty() {
635 if let Some(log) = self.framing_log() {
636 if !tip.queued.is_empty() {
637 let last = tip.unfiled.last().map(|(s, _)| *s).unwrap_or(0);
638 log.sync_through(last).map_err(|e| AuditError::Frame {
639 path: self.path.display().to_string(),
640 detail: e.to_string(),
641 })?;
642 }
643 self.file_durable(tip, &log)?;
644 }
645 if !tip.unfiled.is_empty() && !tip.queued.is_empty() {
646 return Ok(0);
647 }
648 }
649 let mut n = 0;
650 while let Some(line) = tip.queued.first().cloned() {
651 match self.write_line(&line)? {
652 Wrote::Yes => {
653 tip.queued.remove(0);
654 n += 1;
655 }
656 Wrote::Released | Wrote::Follower => break,
657 }
658 }
659 Ok(n)
660 }
661
662 /// Frame `line` (a framed chain) and append it to the file, durable.
663 fn write_line(&self, line: &str) -> Result<Wrote, AuditError> {
664 // **The trail is a transaction before it is a line** (lane T11). Framed FIRST and held
665 // until the file has the write, exactly as `attest.rs` frames an attestation: the
666 // mutation gate keeps a snapshot from cutting between the frame and the byte, so the
667 // twin can never hold a trail that is ahead of the file or behind it.
668 //
669 // NOT best effort. The gate's contract is that a failure here is one `error!` and a
670 // served RPC, and that contract is kept by the CALLER — what must not happen is a line
671 // in the file with no frame behind it, because the restore would then bring back a
672 // chain with a hole in the middle and `verify` would report the break as the
673 // primary's, months later, to somebody who cannot know it was ours.
674 let _framed = if self.framed {
675 match crate::txlog::sink::append_line(&self.path, line) {
676 Ok(f) => Some(f),
677 Err(crate::txlog::Error::Released(_)) => return Ok(Wrote::Released),
678 Err(crate::txlog::Error::Follower) => return Ok(Wrote::Follower),
679 Err(source) => {
680 return Err(AuditError::Frame {
681 path: self.path.display().to_string(),
682 detail: source.to_string(),
683 })
684 }
685 }
686 } else {
687 None
688 };
689 // G3: an UNFRAMED chain (the twin's own trail) has no log to tell it the set is off;
690 // the directory it lives in is gone while a growth has /data unmounted. That is the
691 // same state as a released log: hold the line, write it after the remount — and never
692 // create the directory on the root filesystem under the empty mountpoint.
693 if _framed.is_none() && self.path.parent().is_some_and(|p| !p.exists()) {
694 return Ok(Wrote::Released);
695 }
696 let io = |source: std::io::Error| AuditError::Io {
697 path: self.path.display().to_string(),
698 source,
699 };
700 let mut file = std::fs::OpenOptions::new()
701 .create(true)
702 .append(true)
703 .open(&self.path)
704 .map_err(io)?;
705 file.write_all(format!("{line}\n").as_bytes()).map_err(io)?;
706 // Durable before the caller is told it landed; see the module docs.
707 file.sync_all().map_err(io)?;
708 Ok(Wrote::Yes)
709 }
710
711 /// Every entry, oldest first.
712 pub fn entries(&self) -> Result<Vec<AuditEntry>, AuditError> {
713 read_entries(&self.path)
714 }
715
716 /// **Check the chain end to end.** `Ok(n)` is "n entries, and they hold".
717 ///
718 /// This is the function the finding asks for: an operator (or
719 /// `cargo test`) can ask the file whether anything in it has been changed
720 /// or removed, and get an answer that names the entry rather than a
721 /// shrug.
722 pub fn verify(&self) -> Result<usize, AuditError> {
723 let entries = read_entries(&self.path)?;
724 verify_entries(&entries)?;
725 Ok(entries.len())
726 }
727
728 /// The current tip hash — what a whole history commits to.
729 ///
730 /// Worth writing down somewhere the box cannot reach: with a recorded tip,
731 /// even a rewrite of the entire file is detected.
732 pub fn tip(&self) -> String {
733 self.tip
734 .lock()
735 .unwrap_or_else(|poisoned| poisoned.into_inner())
736 .hash
737 .clone()
738 }
739}
740
741/// Read the file, or `[]` if it is not there yet.
742fn read_entries(path: &Path) -> Result<Vec<AuditEntry>, AuditError> {
743 let text = match std::fs::read_to_string(path) {
744 Ok(text) => text,
745 Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()),
746 Err(source) => {
747 return Err(AuditError::Io {
748 path: path.display().to_string(),
749 source,
750 })
751 }
752 };
753 let mut out = Vec::new();
754 for (index, line) in text.lines().enumerate() {
755 if line.trim().is_empty() {
756 continue;
757 }
758 let entry = serde_json::from_str(line).map_err(|source| AuditError::Malformed {
759 line: index + 1,
760 source,
761 })?;
762 out.push(entry);
763 }
764 Ok(out)
765}
766
767/// The arithmetic, over entries already parsed.
768///
769/// Separate from the file so a caller with entries in hand — a test, a future
770/// reader that streams — checks them with the same code the server does (one
771/// implementation).
772pub fn verify_entries(entries: &[AuditEntry]) -> Result<(), AuditError> {
773 let mut expected_prev = GENESIS.to_owned();
774 for (index, entry) in entries.iter().enumerate() {
775 let line = index + 1;
776 let broken = |what: String| AuditError::Broken {
777 seq: entry.seq,
778 line,
779 what,
780 };
781 if entry.seq != index as u64 {
782 return Err(broken(format!(
783 "it is the {}th entry in the file and claims seq {}; an entry has been removed \
784 or reordered",
785 index, entry.seq
786 )));
787 }
788 if entry.prev != expected_prev {
789 return Err(broken(format!(
790 "it names {} as the entry before it and the entry before it hashes to {}; \
791 the chain has been cut",
792 short(&entry.prev),
793 short(&expected_prev)
794 )));
795 }
796 let computed = entry.compute_hash();
797 if entry.hash != computed {
798 return Err(broken(format!(
799 "its contents hash to {} and it carries {}; this entry has been edited",
800 short(&computed),
801 short(&entry.hash)
802 )));
803 }
804 expected_prev = entry.hash.clone();
805 }
806 Ok(())
807}
808
809/// A hash, shortened for a message a person reads. The full value is in the
810/// file.
811fn short(hash: &str) -> &str {
812 hash.get(..12).unwrap_or(hash)
813}
814
815#[cfg(test)]
816mod tests {
817 use super::*;
818
819 fn event<'a>(method: &'a str, decision: &'a str) -> AuditEvent<'a> {
820 AuditEvent {
821 method,
822 principal: "alice",
823 fingerprint: "SHA256:abc",
824 scope: "repo:team/app",
825 level: "admin",
826 auth_mode: "required",
827 decision,
828 reason: "",
829 }
830 }
831
832 fn chain_of(path: &Path, count: usize) -> AuditChain {
833 let chain = AuditChain::open(path).expect("open");
834 for index in 0..count {
835 chain
836 .append(
837 event("Access.Grant", "allow"),
838 1_700_000_000_000 + index as i64,
839 )
840 .expect("append");
841 }
842 chain
843 }
844
845 /// The accept side, and the shape. Without it every tamper assertion below
846 /// would pass against a `verify` that refused everything.
847 #[test]
848 fn a_chain_links_each_entry_to_the_one_before_it_and_verifies() {
849 let dir = tempfile::tempdir().expect("tempdir");
850 let path = dir.path().join(AUDIT_FILE);
851 let chain = chain_of(&path, 3);
852 assert_eq!(chain.verify().expect("a fresh chain verifies"), 3);
853
854 let entries = chain.entries().expect("read");
855 assert_eq!(entries[0].prev, GENESIS, "the first entry starts the chain");
856 assert_eq!(entries[1].prev, entries[0].hash);
857 assert_eq!(entries[2].prev, entries[1].hash);
858 assert_eq!(chain.tip(), entries[2].hash);
859 assert_eq!((entries[0].seq, entries[2].seq), (0, 2));
860
861 // Reopening continues the chain rather than starting a second one: the
862 // tip is read back off the file, which is what makes a restart
863 // invisible to a verifier.
864 let reopened = AuditChain::open(&path).expect("reopen");
865 assert_eq!(reopened.tip(), entries[2].hash);
866 reopened
867 .append(event("Repos.Delete", "deny"), 1_700_000_009_000)
868 .expect("append after reopen");
869 assert_eq!(reopened.verify().expect("still holds"), 4);
870 let entries = reopened.entries().expect("read");
871 assert_eq!(entries[3].prev, entries[2].hash);
872 }
873
874 /// **A modified entry is detected.** The whole point: `tracing` could not
875 /// say this, and this is the sentence the operator gets.
876 #[test]
877 fn an_edited_entry_is_named() {
878 let dir = tempfile::tempdir().expect("tempdir");
879 let path = dir.path().join(AUDIT_FILE);
880 let _chain = chain_of(&path, 3);
881
882 // Rewrite the middle record's decision, the edit somebody covering a
883 // refusal would make, leaving everything else — the hash included —
884 // exactly as gunnar wrote it.
885 let text = std::fs::read_to_string(&path).expect("read");
886 let mut lines: Vec<String> = text.lines().map(str::to_owned).collect();
887 lines[1] = lines[1].replace("\"decision\":\"allow\"", "\"decision\":\"deny\"");
888 std::fs::write(&path, lines.join("\n") + "\n").expect("rewrite");
889
890 let err = AuditChain::open(&path).expect_err("a rewritten chain must not open");
891 match err {
892 AuditError::Broken {
893 seq,
894 line,
895 ref what,
896 } => {
897 assert_eq!((seq, line), (1, 2), "the break must be named: {what}");
898 assert!(what.contains("edited"), "{what}");
899 }
900 other => panic!("the edit was not detected: {other}"),
901 }
902 }
903
904 /// **A removed entry is detected** — the tamper `tracing` is least able to
905 /// show, because a deleted line leaves nothing behind.
906 #[test]
907 fn a_removed_entry_is_named() {
908 let dir = tempfile::tempdir().expect("tempdir");
909 let path = dir.path().join(AUDIT_FILE);
910 let chain = chain_of(&path, 4);
911 assert_eq!(chain.verify().expect("green first"), 4);
912
913 let text = std::fs::read_to_string(&path).expect("read");
914 let lines: Vec<&str> = text.lines().collect();
915 // Drop the second of four.
916 let cut: Vec<&str> = lines
917 .iter()
918 .enumerate()
919 .filter(|(index, _)| *index != 1)
920 .map(|(_, line)| *line)
921 .collect();
922 std::fs::write(&path, cut.join("\n") + "\n").expect("rewrite");
923
924 let err = AuditChain::open(&path).expect_err("a cut chain must not open");
925 assert!(
926 matches!(
927 err,
928 AuditError::Broken {
929 seq: 2,
930 line: 2,
931 ..
932 }
933 ),
934 "the removal was not detected where it happened: {err}"
935 );
936 }
937
938 /// **Truncating the tail is detected too — but only against a recorded
939 /// tip**, and the difference is stated rather than assumed.
940 ///
941 /// Cutting the LAST entries leaves a chain that is internally consistent:
942 /// that is what a hash chain is, and claiming otherwise would be the kind
943 /// of overclaim this file is here to replace. What catches it is the tip an
944 /// operator wrote down, so the test asserts exactly that — the shortened
945 /// file verifies, and its tip is not the one that was recorded.
946 #[test]
947 fn a_truncated_tail_verifies_but_does_not_match_the_recorded_tip() {
948 let dir = tempfile::tempdir().expect("tempdir");
949 let path = dir.path().join(AUDIT_FILE);
950 let chain = chain_of(&path, 4);
951 let recorded = chain.tip();
952
953 let text = std::fs::read_to_string(&path).expect("read");
954 let kept: Vec<&str> = text.lines().take(2).collect();
955 std::fs::write(&path, kept.join("\n") + "\n").expect("rewrite");
956
957 let shortened = AuditChain::open(&path).expect("a truncated chain is self-consistent");
958 assert_eq!(shortened.verify().expect("and verifies"), 2);
959 assert_ne!(
960 shortened.tip(),
961 recorded,
962 "a truncated chain reproduced the tip of the full one"
963 );
964 }
965
966 /// A field cannot be moved across a boundary into another field, because
967 /// each one is length-prefixed. Without the prefixes these two entries hash
968 /// the same.
969 #[test]
970 fn a_field_holding_a_separator_cannot_impersonate_two_fields() {
971 let base = AuditEntry {
972 seq: 0,
973 at_unix_ms: 1,
974 method: "Access.Grant".into(),
975 principal: "alice".into(),
976 fingerprint: "SHA256:abc".into(),
977 scope: "server".into(),
978 level: "admin".into(),
979 auth_mode: "required".into(),
980 decision: "deny".into(),
981 reason: String::new(),
982 prev: GENESIS.into(),
983 hash: String::new(),
984 };
985 // The pair that COLLIDES under a newline-joined concatenation: both
986 // sides render as `alice\nSHA256:abc\n`, and only the length prefixes
987 // tell them apart.
988 let mut shifted = base.clone();
989 shifted.principal = "alice\nSHA256:abc".into();
990 shifted.fingerprint = String::new();
991 // The naive concatenation of the two differs only in where the
992 // boundary is drawn.
993 assert_ne!(
994 base.compute_hash(),
995 shifted.compute_hash(),
996 "two entries with different field boundaries hash the same"
997 );
998 }
999
1000 // ── the trail survives a restore (lane T11, 2026-09-21; the D6 finding) ──
1001
1002 /// A data root with a transaction log registered on it, and the audit chain where the
1003 /// layout puts it: `<root>/log/audit-chain.jsonl`.
1004 fn framed_root() -> (
1005 tempfile::TempDir,
1006 std::sync::Arc<crate::txlog::Log>,
1007 PathBuf,
1008 ) {
1009 let dir = tempfile::tempdir().expect("tempdir");
1010 let log =
1011 std::sync::Arc::new(crate::txlog::Log::open(dir.path().join("txlog")).expect("a log"));
1012 crate::txlog::sink::register(dir.path(), &log);
1013 let path = dir.path().join(crate::txlog::AUDIT_REL);
1014 (dir, log, path)
1015 }
1016
1017 fn a_decision(method: &str) -> AuditEvent<'_> {
1018 AuditEvent {
1019 method,
1020 principal: "alice",
1021 fingerprint: "SHA256:abc",
1022 scope: "server",
1023 level: "admin",
1024 auth_mode: "required",
1025 decision: "allow",
1026 reason: "",
1027 }
1028 }
1029
1030 /// **Every entry of a framed chain is a transaction**, so the twin holds the audit trail
1031 /// and `gunnar control twin replay` brings it back. That is the whole D6 fix: before it,
1032 /// `log/` was a per-box path and a restore came up with no audit history, silently.
1033 ///
1034 /// Proven the way a restore would see it: the frames are read back out of the log and the
1035 /// file is rebuilt from them alone, then opened and verified.
1036 ///
1037 /// **Fails before the fix**: make `AuditChain::open_framed` return the chain that
1038 /// `AuditChain::open` returns (drop the `chain.framed = true;` line) and the log holds
1039 /// nothing — `head_seq()` is 0 and the rebuilt file is empty.
1040 #[test]
1041 fn a_framed_chain_puts_every_entry_in_the_transaction_log() {
1042 let (_dir, log, path) = framed_root();
1043 let chain = AuditChain::open_framed(&path).expect("the trail is framable where it lives");
1044 assert!(chain.is_framed());
1045 for m in ["Repos.Create", "Access.Grant", "Policy.SetProtection"] {
1046 chain.append(a_decision(m), 1).expect("append");
1047 }
1048 assert_eq!(chain.verify().expect("verify"), 3);
1049
1050 // What the twin holds: three `Append` frames, all naming the ONE canonical path, in
1051 // the order the chain wrote them.
1052 assert_eq!(log.head_seq(), 3, "one frame per decision");
1053 let mut rebuilt = String::new();
1054 for seq in 1..=3 {
1055 let e = log.read(seq).expect("a frame");
1056 assert_eq!(e.kind(), Some(crate::txlog::Kind::Append));
1057 match e.meta().expect("meta") {
1058 crate::txlog::Meta::Append { log: rel } => assert_eq!(rel, crate::txlog::AUDIT_REL),
1059 other => panic!("seq {seq} is {other:?}, not an Append of the audit trail"),
1060 }
1061 rebuilt.push_str(std::str::from_utf8(&e.blob).expect("utf8"));
1062 rebuilt.push('\n');
1063 }
1064
1065 // A restored box rebuilds the file from the frames alone — and it verifies, with the
1066 // same tip. That is the property the owner asked for in one assertion.
1067 let elsewhere = tempfile::tempdir().expect("tempdir");
1068 let restored_path = elsewhere.path().join("audit-chain.jsonl");
1069 std::fs::write(&restored_path, rebuilt).expect("write");
1070 let restored = AuditChain::open(&restored_path).expect("a restored trail opens");
1071 assert_eq!(restored.verify().expect("verify"), 3);
1072 assert_eq!(restored.tip(), chain.tip());
1073
1074 // And the restored box CONTINUES the chain rather than starting a new one: its next
1075 // entry's `prev` is the dead primary's tip, so the history is unbroken across the
1076 // disaster.
1077 let next = restored
1078 .append(a_decision("Repos.List"), 2)
1079 .expect("append");
1080 assert_eq!(next.prev, chain.tip());
1081 assert_eq!(next.seq, 3);
1082 }
1083
1084 /// **A chain that cannot be replicated REFUSES to pretend it is**, rather than quietly
1085 /// writing unframed lines — which is the exact shape of the bug being fixed: a record the
1086 /// restore drops without saying so.
1087 ///
1088 /// Two ways it can happen: the `log` directory resolved outside any data root (no sink),
1089 /// and the chain sitting somewhere other than the one path a restore resolves.
1090 ///
1091 /// **Fails before the fix**: delete the `match crate::txlog::sink::log_for(&path)` block
1092 /// from `AuditChain::open_framed` and both halves return `Ok`.
1093 #[test]
1094 fn an_unreplicable_chain_refuses_to_be_the_replicated_trail() {
1095 let outside = tempfile::tempdir().expect("tempdir");
1096 let err = AuditChain::open_framed(outside.path().join("audit-chain.jsonl"))
1097 .expect_err("a chain under no data root is not the replicated trail");
1098 assert!(matches!(err, AuditError::Unframed { .. }), "{err}");
1099 assert!(format!("{err}").contains("under no data root"), "{err}");
1100
1101 let (dir, _log, _path) = framed_root();
1102 let misplaced = dir.path().join("somewhere-else.jsonl");
1103 let err = AuditChain::open_framed(&misplaced).expect_err("a misplaced chain is refused");
1104 assert!(matches!(err, AuditError::Unframed { .. }), "{err}");
1105 assert!(format!("{err}").contains(crate::txlog::AUDIT_REL), "{err}");
1106 }
1107
1108 /// **An unframed chain frames nothing** — the twin's own trail
1109 /// ([`crate::txlog::AUDIT_LOCAL_REL`]) and every test harness. A twin originates nothing,
1110 /// and a twin that put its own decisions into the log it is holding for the primary would
1111 /// be forging the primary's trail.
1112 ///
1113 /// **Fails before the fix**: make `AuditChain::append` frame unconditionally (drop the
1114 /// `if self.framed`) and the log holds the twin's own decision at seq 1.
1115 #[test]
1116 fn a_twins_own_trail_is_never_a_transaction() {
1117 let (dir, log, _path) = framed_root();
1118 let local = dir.path().join(crate::txlog::AUDIT_LOCAL_REL);
1119 let chain = AuditChain::open(&local).expect("the twin's own trail opens");
1120 assert!(!chain.is_framed());
1121 chain
1122 .append(a_decision("Mirror.LogHead"), 1)
1123 .expect("append");
1124 assert_eq!(
1125 log.head_seq(),
1126 0,
1127 "a twin originates nothing into the primary's log"
1128 );
1129 assert_eq!(chain.verify().expect("verify"), 1);
1130 let _ = log;
1131 }
1132
1133 /// **G (release-3 test2 mock 2026-09-29): a box that FOLLOWS originates nothing into the
1134 /// replicated trail, and says so without an error.** The twin opened its chain framed (it
1135 /// started as a primary) and was then told `Bootstrap.BecomeTwin` at runtime: its log turned
1136 /// follower, and every later decision (`Admin.GrowStatus` polls from monetize, 13 of them)
1137 /// was an `ERROR txlog: this box is a twin (or restoring); it originates no transactions`.
1138 ///
1139 /// A follower's decision is NOT written — not into the log (the primary's numbers), and not
1140 /// into the file (the primary's chain, which the twin holds) — and the answer is a
1141 /// [`Recorded::NotOriginated`], not an error. A box promoted again continues from the tip the
1142 /// file holds THEN, not from the one it remembered before it followed.
1143 #[test]
1144 fn a_framed_chain_on_a_box_that_follows_originates_nothing_and_is_no_error() {
1145 let (_dir, log, path) = framed_root();
1146 let chain = AuditChain::open_framed(&path).expect("framed");
1147 chain
1148 .append(a_decision("Repos.Create"), 1)
1149 .expect("the primary's own decision");
1150 log.set_follower(true);
1151 match chain
1152 .record(a_decision("Admin.GrowStatus"), 2)
1153 .expect("a follower's decision is not an error")
1154 {
1155 Recorded::NotOriginated => {}
1156 other => panic!("a follower originates nothing: {other:?}"),
1157 }
1158 assert!(matches!(
1159 chain.append(a_decision("Admin.GrowStatus"), 3),
1160 Err(AuditError::NotOriginated { .. })
1161 ));
1162 assert_eq!(
1163 log.head_seq(),
1164 1,
1165 "nothing framed into the primary's numbers"
1166 );
1167 assert_eq!(
1168 chain.verify().expect("verify"),
1169 1,
1170 "nothing written into the primary's chain"
1171 );
1172
1173 // The primary's own frames reach the file while it follows (the twin applies them) …
1174 let other = AuditChain::open(&path).expect("the applier's view");
1175 let applied = other
1176 .append(a_decision("Access.Grant"), 4)
1177 .expect("applied");
1178 // … and a promoted box continues from THAT tip.
1179 log.set_follower(false);
1180 let next = chain
1181 .append(a_decision("Repos.List"), 5)
1182 .expect("originates again");
1183 assert_eq!(
1184 (next.seq, next.prev.clone()),
1185 (applied.seq + 1, applied.hash.clone())
1186 );
1187 assert_eq!(chain.verify().expect("verify"), 3);
1188 }
1189
1190 /// **G2: while the data set is grown the trail WAITS, in order, and is written after the
1191 /// remount — never an error.** MEASURED main on the same mock: 4 `ERROR txlog: released — the
1192 /// files under /data/txlog are given up while the data set is grown` on `Admin.Capacity` /
1193 /// `Admin.GrowStatus` during its growth, and those decisions were never in the trail.
1194 #[test]
1195 fn a_released_log_queues_the_trail_and_the_first_append_after_the_remount_writes_it_in_order() {
1196 let (_dir, log, path) = framed_root();
1197 let chain = AuditChain::open_framed(&path).expect("framed");
1198 chain.append(a_decision("Repos.Create"), 1).expect("append");
1199 log.release().expect("the grow releases the log");
1200 for (i, m) in ["Admin.Capacity", "Admin.GrowStatus"]
1201 .into_iter()
1202 .enumerate()
1203 {
1204 match chain
1205 .record(a_decision(m), 2 + i as i64)
1206 .expect("a decision during the grow is not an error")
1207 {
1208 Recorded::Queued { pending, .. } => assert_eq!(pending, i + 1),
1209 other => panic!("released: queued, not {other:?}"),
1210 }
1211 }
1212 assert_eq!(chain.queued(), 2);
1213 assert_eq!(
1214 log.head_seq(),
1215 1,
1216 "nothing framed while the files are given up"
1217 );
1218 assert_eq!(
1219 chain.verify().expect("verify"),
1220 1,
1221 "nothing written under the unmounted set"
1222 );
1223 log.reacquire().expect("the remount");
1224 match chain
1225 .record(a_decision("Admin.GrowStatus"), 9)
1226 .expect("after the remount")
1227 {
1228 Recorded::Written(e) => assert_eq!(e.seq, 3),
1229 other => panic!("written after the remount: {other:?}"),
1230 }
1231 assert_eq!(chain.queued(), 0);
1232 assert_eq!(chain.verify().expect("the chain holds across the grow"), 4);
1233 assert_eq!(log.head_seq(), 4, "every decision framed, in order");
1234 let methods: Vec<String> = chain
1235 .entries()
1236 .unwrap()
1237 .into_iter()
1238 .map(|e| e.method)
1239 .collect();
1240 assert_eq!(
1241 methods,
1242 [
1243 "Repos.Create",
1244 "Admin.Capacity",
1245 "Admin.GrowStatus",
1246 "Admin.GrowStatus"
1247 ]
1248 );
1249 // `flush` writes a queue without waiting for the next decision.
1250 log.release().unwrap();
1251 chain.record(a_decision("Admin.Capacity"), 10).unwrap();
1252 log.reacquire().unwrap();
1253 assert_eq!(chain.flush().expect("flush"), 1);
1254 assert_eq!(
1255 (chain.queued(), chain.verify().unwrap(), log.head_seq()),
1256 (0, 5, 5)
1257 );
1258 }
1259
1260 /// **G3 (d-upg3 2026-10-02, the KVM mock of the full release upgrade at ops fa23a53): an
1261 /// UNFRAMED chain whose set is unmounted for a growth WAITS too — never an error.** With
1262 /// `[machines.twin]` fed to vm1, monetize grows the TWIN; the twin's own trail
1263 /// ([`crate::txlog::AUDIT_LOCAL_REL`]) is not framed, so nothing told it the set was off, and
1264 /// monetize's own `Admin.GrowStatus` poll during the growth answered `ERROR … the audit chain
1265 /// at /data/log/audit-chain.local.jsonl: No such file or directory` — the log gate's FAIL.
1266 /// The directory the chain lives in is GONE while /data is unmounted: the line is held, in
1267 /// order, and written by the first decision after the remount (or `flush`).
1268 #[test]
1269 fn an_unframed_chain_whose_set_is_unmounted_queues_and_writes_after_the_remount() {
1270 let dir = tempfile::tempdir().unwrap();
1271 let data = dir.path().join("data");
1272 let local = data.join(crate::txlog::AUDIT_LOCAL_REL);
1273 let chain = AuditChain::open(&local).expect("the twin's own trail opens");
1274 chain
1275 .append(a_decision("Mirror.LogHead"), 1)
1276 .expect("before the grow");
1277 // The growth unmounts /data: the directory the chain is in is not there.
1278 let off = dir.path().join("data-off");
1279 std::fs::rename(&data, &off).unwrap();
1280 match chain
1281 .record(a_decision("Admin.GrowStatus"), 2)
1282 .expect("a decision while the set is off is not an error")
1283 {
1284 Recorded::Queued { pending, .. } => assert_eq!(pending, 1),
1285 other => panic!("the set is off: queued, not {other:?}"),
1286 }
1287 assert!(
1288 !data.exists(),
1289 "nothing is created where the set is not mounted"
1290 );
1291 std::fs::rename(&off, &data).unwrap();
1292 match chain
1293 .record(a_decision("Admin.GrowStatus"), 3)
1294 .expect("after the remount")
1295 {
1296 Recorded::Written(e) => assert_eq!(e.seq, 2),
1297 other => panic!("written after the remount: {other:?}"),
1298 }
1299 assert_eq!(
1300 (
1301 chain.queued(),
1302 chain.verify().expect("the chain holds across the grow")
1303 ),
1304 (0, 3)
1305 );
1306 }
1307
1308 /// **Decisions group-committed from many threads, with a snapshot's freeze cutting in,
1309 /// never deadlock and never fork the chain** (perf, 2026-10-06). Eight threads record while
1310 /// another freezes the log over and over: every decision lands once, the chain verifies, the
1311 /// file holds exactly the lines the log's frames carry, in the same order — and the whole run
1312 /// ends (a thread that held the chain's lock while it waited on the freeze would stop the
1313 /// thread the freeze waits for; the guard is taken before the lock).
1314 #[test]
1315 fn concurrent_framed_decisions_with_freezes_land_once_in_order_and_never_deadlock() {
1316 let (_dir, log, path) = framed_root();
1317 let chain = std::sync::Arc::new(AuditChain::open_framed(&path).expect("framed"));
1318 let (tx, rx) = std::sync::mpsc::channel();
1319 {
1320 let (chain, log) = (std::sync::Arc::clone(&chain), std::sync::Arc::clone(&log));
1321 std::thread::spawn(move || {
1322 let stop = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
1323 let freezer = {
1324 let (log, stop) = (std::sync::Arc::clone(&log), std::sync::Arc::clone(&stop));
1325 std::thread::spawn(move || {
1326 let mut n = 0u64;
1327 while !stop.load(std::sync::atomic::Ordering::SeqCst) {
1328 let g = log.freeze();
1329 std::thread::sleep(std::time::Duration::from_micros(200));
1330 drop(g);
1331 n += 1;
1332 }
1333 n
1334 })
1335 };
1336 std::thread::scope(|s| {
1337 for t in 0..8 {
1338 let chain = std::sync::Arc::clone(&chain);
1339 s.spawn(move || {
1340 for i in 0..40 {
1341 chain
1342 .append(
1343 a_decision(if (t + i) % 2 == 0 {
1344 "Repos.Create"
1345 } else {
1346 "Access.Grant"
1347 }),
1348 i,
1349 )
1350 .expect("a decision lands");
1351 }
1352 });
1353 }
1354 });
1355 stop.store(true, std::sync::atomic::Ordering::SeqCst);
1356 let _ = tx.send(freezer.join().unwrap());
1357 });
1358 }
1359 let freezes = rx
1360 .recv_timeout(std::time::Duration::from_secs(120))
1361 .expect("320 decisions under freezes end — no deadlock");
1362 assert!(freezes > 0, "the freezer cut in");
1363 assert_eq!(chain.verify().expect("the chain holds"), 320);
1364 let file: Vec<String> = std::fs::read_to_string(&path)
1365 .unwrap()
1366 .lines()
1367 .map(str::to_owned)
1368 .collect();
1369 let framed: Vec<String> = (1..=log.head_seq())
1370 .map(|s| String::from_utf8(log.read(s).unwrap().blob).unwrap())
1371 .collect();
1372 assert_eq!(
1373 file, framed,
1374 "the file is exactly the log's frames, in the log's order"
1375 );
1376 }
1377}